{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: You are using pip version 20.2.2; however, version 20.2.3 is available.\r\n",
      "You should consider upgrading via the '/opt/conda/bin/python3.7 -m pip install --upgrade pip' command.\u001b[0m\r\n"
     ]
    }
   ],
   "source": [
    "!pip install -q transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m W&B installed but not logged in.  Run `wandb login` or set the WANDB_API_KEY env variable.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "from pathlib import Path \n",
    "\n",
    "import os\n",
    "\n",
    "import torch\n",
    "import torch.optim as optim\n",
    "\n",
    "import random \n",
    "\n",
    "# fastai\n",
    "from fastai import *\n",
    "from fastai.text import *\n",
    "from fastai.callbacks import *\n",
    "\n",
    "# transformers\n",
    "from transformers import PreTrainedModel, PreTrainedTokenizer, PretrainedConfig\n",
    "\n",
    "from transformers import BertForSequenceClassification, BertTokenizer, BertConfig\n",
    "from transformers import RobertaForSequenceClassification, RobertaTokenizer, RobertaConfig\n",
    "from transformers import XLNetForSequenceClassification, XLNetTokenizer, XLNetConfig\n",
    "from transformers import XLMForSequenceClassification, XLMTokenizer, XLMConfig\n",
    "from transformers import DistilBertForSequenceClassification, DistilBertTokenizer, DistilBertConfig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fastai version : 1.0.61\n",
      "transformers version : 2.11.0\n"
     ]
    }
   ],
   "source": [
    "import fastai\n",
    "import transformers\n",
    "print('fastai version :', fastai.__version__)\n",
    "print('transformers version :', transformers.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/kaggle/input/mh-20-fakenews/Participants_Data_WH20/Test.csv\n",
      "/kaggle/input/mh-20-fakenews/Participants_Data_WH20/Train.csv\n",
      "/kaggle/input/mh-20-fakenews/Participants_Data_WH20/sample submission.csv\n"
     ]
    }
   ],
   "source": [
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10240, 3) (1267, 2)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Labels</th>\n",
       "      <th>Text</th>\n",
       "      <th>Text_Tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Says the Annies List political group supports ...</td>\n",
       "      <td>abortion</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>When did the decline of coal start? It started...</td>\n",
       "      <td>energy,history,job-accomplishments</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Hillary Clinton agrees with John McCain \"by vo...</td>\n",
       "      <td>foreign-policy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>Health care reform legislation is likely to ma...</td>\n",
       "      <td>health-care</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>The economic turnaround started at the end of ...</td>\n",
       "      <td>economy,jobs</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Labels                                               Text  \\\n",
       "0       1  Says the Annies List political group supports ...   \n",
       "1       2  When did the decline of coal start? It started...   \n",
       "2       3  Hillary Clinton agrees with John McCain \"by vo...   \n",
       "3       1  Health care reform legislation is likely to ma...   \n",
       "4       2  The economic turnaround started at the end of ...   \n",
       "\n",
       "                             Text_Tag  \n",
       "0                            abortion  \n",
       "1  energy,history,job-accomplishments  \n",
       "2                      foreign-policy  \n",
       "3                         health-care  \n",
       "4                        economy,jobs  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('../input/mh-20-fakenews/Participants_Data_WH20/Train.csv')\n",
    "test = pd.read_csv('../input/mh-20-fakenews/Participants_Data_WH20/Test.csv')\n",
    "print(train.shape,test.shape)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# remove nan in train\n",
    "train = train.drop(index=2142)\n",
    "train = train.drop(index=9375)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "MODEL_CLASSES = {\n",
    "    'bert': (BertForSequenceClassification, BertTokenizer, BertConfig),\n",
    "    'xlnet': (XLNetForSequenceClassification, XLNetTokenizer, XLNetConfig),\n",
    "    'xlm': (XLMForSequenceClassification, XLMTokenizer, XLMConfig),\n",
    "    'roberta': (RobertaForSequenceClassification, RobertaTokenizer, RobertaConfig),\n",
    "    'distilbert': (DistilBertForSequenceClassification, DistilBertTokenizer, DistilBertConfig)\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Parameters\n",
    "seed = 42\n",
    "use_fp16 = False\n",
    "bs = 16\n",
    "\n",
    "model_type = 'roberta'\n",
    "pretrained_model_name = 'roberta-base'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_class, tokenizer_class, config_class = MODEL_CLASSES[model_type]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def seed_all(seed_value):\n",
    "    random.seed(seed_value) # Python\n",
    "    np.random.seed(seed_value) # cpu vars\n",
    "    torch.manual_seed(seed_value) # cpu  vars\n",
    "    \n",
    "    if torch.cuda.is_available(): \n",
    "        torch.cuda.manual_seed(seed_value)\n",
    "        torch.cuda.manual_seed_all(seed_value) # gpu vars\n",
    "        torch.backends.cudnn.deterministic = True  #needed\n",
    "        torch.backends.cudnn.benchmark = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "seed_all(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class TransformersBaseTokenizer(BaseTokenizer):\n",
    "    \"\"\"Wrapper around PreTrainedTokenizer to be compatible with fast.ai\"\"\"\n",
    "    def __init__(self, pretrained_tokenizer: PreTrainedTokenizer, model_type = 'bert', **kwargs):\n",
    "        self._pretrained_tokenizer = pretrained_tokenizer\n",
    "        self.max_seq_len = pretrained_tokenizer.max_len\n",
    "        self.model_type = model_type\n",
    "\n",
    "    def __call__(self, *args, **kwargs): \n",
    "        return self\n",
    "\n",
    "    def tokenizer(self, t:str) -> List[str]:\n",
    "        \"\"\"Limits the maximum sequence length and add the spesial tokens\"\"\"\n",
    "        CLS = self._pretrained_tokenizer.cls_token\n",
    "        SEP = self._pretrained_tokenizer.sep_token\n",
    "        if self.model_type in ['roberta']:\n",
    "            tokens = self._pretrained_tokenizer.tokenize(t, add_prefix_space=True)[:self.max_seq_len - 2]\n",
    "            tokens = [CLS] + tokens + [SEP]\n",
    "        else:\n",
    "            tokens = self._pretrained_tokenizer.tokenize(t)[:self.max_seq_len - 2]\n",
    "            if self.model_type in ['xlnet']:\n",
    "                tokens = tokens + [SEP] +  [CLS]\n",
    "            else:\n",
    "                tokens = [CLS] + tokens + [SEP]\n",
    "        return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f78b327fb9ba45a5998fbd66c563d86f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=898823.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a078878ef9b34cf4bacc163f45002777",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=456318.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "transformer_tokenizer = tokenizer_class.from_pretrained(pretrained_model_name)\n",
    "transformer_base_tokenizer = TransformersBaseTokenizer(pretrained_tokenizer = transformer_tokenizer, model_type = model_type)\n",
    "fastai_tokenizer = Tokenizer(tok_func = transformer_base_tokenizer, pre_rules=[], post_rules=[])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class TransformersVocab(Vocab):\n",
    "    def __init__(self, tokenizer: PreTrainedTokenizer):\n",
    "        super(TransformersVocab, self).__init__(itos = [])\n",
    "        self.tokenizer = tokenizer\n",
    "    \n",
    "    def numericalize(self, t:Collection[str]) -> List[int]:\n",
    "        \"Convert a list of tokens `t` to their ids.\"\n",
    "        return self.tokenizer.convert_tokens_to_ids(t)\n",
    "        #return self.tokenizer.encode(t)\n",
    "\n",
    "    def textify(self, nums:Collection[int], sep=' ') -> List[str]:\n",
    "        \"Convert a list of `nums` to their tokens.\"\n",
    "        nums = np.array(nums).tolist()\n",
    "        return sep.join(self.tokenizer.convert_ids_to_tokens(nums)) if sep is not None else self.tokenizer.convert_ids_to_tokens(nums)\n",
    "    \n",
    "    def __getstate__(self):\n",
    "        return {'itos':self.itos, 'tokenizer':self.tokenizer}\n",
    "\n",
    "    def __setstate__(self, state:dict):\n",
    "        self.itos = state['itos']\n",
    "        self.tokenizer = state['tokenizer']\n",
    "        self.stoi = collections.defaultdict(int,{v:k for k,v in enumerate(self.itos)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "transformer_vocab =  TransformersVocab(tokenizer = transformer_tokenizer)\n",
    "numericalize_processor = NumericalizeProcessor(vocab=transformer_vocab)\n",
    "\n",
    "tokenize_processor = TokenizeProcessor(tokenizer=fastai_tokenizer, include_bos=False, include_eos=False)\n",
    "\n",
    "transformer_processor = [tokenize_processor, numericalize_processor]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pad_first = bool(model_type in ['xlnet'])\n",
    "pad_idx = transformer_tokenizer.pad_token_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train['combine'] = train['Text']+' about '+ train['Text_Tag']\n",
    "test['combine'] = test['Text']+' about '+ test['Text_Tag']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "databunch = (TextList.from_df(train, cols='combine', processor=transformer_processor)\n",
    "             .split_by_rand_pct(0.1,seed=seed)\n",
    "             .label_from_df(cols= 'Labels')\n",
    "             .add_test(test)\n",
    "             .databunch(bs=bs, pad_first=pad_first, pad_idx=pad_idx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CLS] token : <s>\n",
      "[SEP] token : </s>\n",
      "[PAD] token : <pad>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>&lt;s&gt; ĠHosp itals , Ġdoctors , ĠMR Is , Ġsurgeries Ġand Ġso Ġforth Ġare Ġmore Ġextensively Ġused Ġand Ġfar Ġmore Ġexpensive Ġin Ġthis Ġcountry Ġthan Ġthey Ġare Ġin Ġmany Ġother Ġcountries .'' ĉ health - care ĉ mitt - rom ney ĉ Former Ġgovernor ĉ Mass achusetts ĉ rep ublic an ĉ 34 ĉ 32 ĉ 58 ĉ 33 ĉ 19 ĉ a ĠFox ĠNews ĠSunday Ġinterview Ċ 98 74</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>&lt;s&gt; ĠRhode ĠIsland Ġhas Ġthe Ġsecond Ġhighest Ġper Ġenrol lee Ġ( Medic aid ) Ġcost Ġof Ġany Ġstate Ġin Ġthe Ġcountry , Ġwhich Ġis Ġ60 Ġpercent Ġhigher Ġthan Ġthe Ġnational Ġaverage . Ġabout Ġeconomy , f ederal - budget , government - efficiency , health - care , med ic aid , p overty , public - health , market - regulation , state - budget , state - fin</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>&lt;s&gt; ĠI Ġvetoed Ġmore Ġthan Ġ$ 3 Ġbillion Ġin Ġspending . ĠLaw makers Ġand ĠI Ġcut Ġtaxes Ġfor Ġ40 , 000 Ġsmall Ġbusinesses . ĠNow ĠTexas Ġis ĠNo . Ġ1 Ġin Ġjob Ġcreation . ĠThe Ġnations Ġfive Ġbest Ġhousing Ġmarkets Ġare Ġright Ġhere Ġin ĠTexas . ĠWe Ġbalanced Ġfive Ġstate Ġbudgets Ġand Ġwe Ġcut Ġstate Ġspending . Ġabout Ġeconomy , housing , job - ac compl ish ments , state</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>&lt;s&gt; ĠWhen Ġa ĠGallup Ġpoll Ġasked Ġabout Ġpresidential Ġcandidate Ġcharacteristics , ĠWhen Ġthey Ġget Ġto Ġthe Ġquestion Ġon , ĠWould Ġyou Ġvote Ġfor Ġa Ġsocialist , Ġit Ġis Ġeven Ġless Ġpopular Ġthan Ġvoting Ġfor Ġatheists . Ġabout Ġcandidates - bi ography , g ays - and - les bians , history , campaign - advertising , poll s , p und its , abc - news - week , women</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>&lt;s&gt; ĠI Ġwas Ġspeaking Ġout Ġon Ġpension Ġreform Ġalmost Ġ17 Ġyears Ġago . Ġabout Ġcandidates - bi ography , deb t , def icit , econom y , government - regulation , history , job - ac compl ish ments , l abor , p ensions , ret irement , state - budget , state - fin ances , tax es , un ions , wealth , workers &lt;/s&gt;</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print('[CLS] token :', transformer_tokenizer.cls_token)\n",
    "print('[SEP] token :', transformer_tokenizer.sep_token)\n",
    "print('[PAD] token :', transformer_tokenizer.pad_token)\n",
    "databunch.show_batch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CLS] id : 0\n",
      "[SEP] id : 2\n",
      "[PAD] id : 1\n",
      "Batch shape :  torch.Size([16, 512])\n",
      "tensor([[    0, 13377, 19196,  ...,     6, 15110,     2],\n",
      "        [    0,  1806,   236,  ...,     1,     1,     1],\n",
      "        [    0,   520,    24,  ...,     1,     1,     1],\n",
      "        ...,\n",
      "        [    0, 10747,  3771,  ...,     1,     1,     1],\n",
      "        [    0,  2096,  1172,  ...,     1,     1,     1],\n",
      "        [    0,  1568,  1909,  ...,     1,     1,     1]])\n"
     ]
    }
   ],
   "source": [
    "print('[CLS] id :', transformer_tokenizer.cls_token_id)\n",
    "print('[SEP] id :', transformer_tokenizer.sep_token_id)\n",
    "print('[PAD] id :', pad_idx)\n",
    "test_one_batch = databunch.one_batch()[0]\n",
    "print('Batch shape : ',test_one_batch.shape)\n",
    "print(test_one_batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# defining our model architecture \n",
    "class CustomTransformerModel(nn.Module):\n",
    "    def __init__(self, transformer_model: PreTrainedModel):\n",
    "        super(CustomTransformerModel,self).__init__()\n",
    "        self.transformer = transformer_model\n",
    "        \n",
    "    def forward(self, input_ids, attention_mask=None):\n",
    "        \n",
    "        # attention_mask\n",
    "        # Mask to avoid performing attention on padding token indices.\n",
    "        # Mask values selected in ``[0, 1]``:\n",
    "        # ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens.\n",
    "        attention_mask = (input_ids!=pad_idx).type(input_ids.type()) \n",
    "        \n",
    "        logits = self.transformer(input_ids,\n",
    "                                  attention_mask = attention_mask)[0]   \n",
    "        return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5c9f1f23a60c42658047897ea18bbca2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=481.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "RobertaConfig {\n",
      "  \"architectures\": [\n",
      "    \"RobertaForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"bos_token_id\": 0,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"id2label\": {\n",
      "    \"0\": \"LABEL_0\",\n",
      "    \"1\": \"LABEL_1\",\n",
      "    \"2\": \"LABEL_2\",\n",
      "    \"3\": \"LABEL_3\",\n",
      "    \"4\": \"LABEL_4\",\n",
      "    \"5\": \"LABEL_5\"\n",
      "  },\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"label2id\": {\n",
      "    \"LABEL_0\": 0,\n",
      "    \"LABEL_1\": 1,\n",
      "    \"LABEL_2\": 2,\n",
      "    \"LABEL_3\": 3,\n",
      "    \"LABEL_4\": 4,\n",
      "    \"LABEL_5\": 5\n",
      "  },\n",
      "  \"layer_norm_eps\": 1e-05,\n",
      "  \"max_position_embeddings\": 514,\n",
      "  \"model_type\": \"roberta\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 1,\n",
      "  \"type_vocab_size\": 1,\n",
      "  \"vocab_size\": 50265\n",
      "}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "config = config_class.from_pretrained(pretrained_model_name)\n",
    "config.num_labels = 6\n",
    "config.use_bfloat16 = use_fp16\n",
    "print(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1ed1229d62c24343a89d4b201faf51c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=501200538.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "transformer_model = model_class.from_pretrained(pretrained_model_name, config = config)\n",
    "# transformer_model = model_class.from_pretrained(pretrained_model_name, num_labels = 6)\n",
    "\n",
    "custom_transformer_model = CustomTransformerModel(transformer_model = transformer_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from fastai.callbacks import *\n",
    "from transformers import AdamW\n",
    "from functools import partial\n",
    "\n",
    "CustomAdamW = partial(AdamW, correct_bias=False)\n",
    "\n",
    "learner = Learner(databunch, \n",
    "                  custom_transformer_model, \n",
    "                  opt_func = CustomAdamW, \n",
    "                  metrics=[accuracy, error_rate])\n",
    "\n",
    "# Show graph of learner stats and metrics after each epoch.\n",
    "learner.callbacks.append(ShowGraph(learner))\n",
    "\n",
    "# Put learn in FP16 precision mode. --> Seems to not working\n",
    "if use_fp16: learner = learner.to_fp16()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CustomTransformerModel(\n",
      "  (transformer): RobertaForSequenceClassification(\n",
      "    (roberta): RobertaModel(\n",
      "      (embeddings): RobertaEmbeddings(\n",
      "        (word_embeddings): Embedding(50265, 768, padding_idx=1)\n",
      "        (position_embeddings): Embedding(514, 768, padding_idx=1)\n",
      "        (token_type_embeddings): Embedding(1, 768)\n",
      "        (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout): Dropout(p=0.1, inplace=False)\n",
      "      )\n",
      "      (encoder): BertEncoder(\n",
      "        (layer): ModuleList(\n",
      "          (0): BertLayer(\n",
      "            (attention): BertAttention(\n",
      "              (self): BertSelfAttention(\n",
      "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "              (output): BertSelfOutput(\n",
      "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (intermediate): BertIntermediate(\n",
      "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
      "            )\n",
      "            (output): BertOutput(\n",
      "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
      "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (1): BertLayer(\n",
      "            (attention): BertAttention(\n",
      "              (self): BertSelfAttention(\n",
      "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "              (output): BertSelfOutput(\n",
      "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (intermediate): BertIntermediate(\n",
      "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
      "            )\n",
      "            (output): BertOutput(\n",
      "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
      "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (2): BertLayer(\n",
      "            (attention): BertAttention(\n",
      "              (self): BertSelfAttention(\n",
      "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "              (output): BertSelfOutput(\n",
      "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (intermediate): BertIntermediate(\n",
      "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
      "            )\n",
      "            (output): BertOutput(\n",
      "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
      "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (3): BertLayer(\n",
      "            (attention): BertAttention(\n",
      "              (self): BertSelfAttention(\n",
      "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "              (output): BertSelfOutput(\n",
      "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (intermediate): BertIntermediate(\n",
      "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
      "            )\n",
      "            (output): BertOutput(\n",
      "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
      "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (4): BertLayer(\n",
      "            (attention): BertAttention(\n",
      "              (self): BertSelfAttention(\n",
      "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "              (output): BertSelfOutput(\n",
      "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (intermediate): BertIntermediate(\n",
      "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
      "            )\n",
      "            (output): BertOutput(\n",
      "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
      "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (5): BertLayer(\n",
      "            (attention): BertAttention(\n",
      "              (self): BertSelfAttention(\n",
      "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "              (output): BertSelfOutput(\n",
      "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (intermediate): BertIntermediate(\n",
      "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
      "            )\n",
      "            (output): BertOutput(\n",
      "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
      "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (6): BertLayer(\n",
      "            (attention): BertAttention(\n",
      "              (self): BertSelfAttention(\n",
      "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "              (output): BertSelfOutput(\n",
      "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (intermediate): BertIntermediate(\n",
      "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
      "            )\n",
      "            (output): BertOutput(\n",
      "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
      "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (7): BertLayer(\n",
      "            (attention): BertAttention(\n",
      "              (self): BertSelfAttention(\n",
      "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "              (output): BertSelfOutput(\n",
      "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (intermediate): BertIntermediate(\n",
      "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
      "            )\n",
      "            (output): BertOutput(\n",
      "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
      "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (8): BertLayer(\n",
      "            (attention): BertAttention(\n",
      "              (self): BertSelfAttention(\n",
      "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "              (output): BertSelfOutput(\n",
      "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (intermediate): BertIntermediate(\n",
      "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
      "            )\n",
      "            (output): BertOutput(\n",
      "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
      "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (9): BertLayer(\n",
      "            (attention): BertAttention(\n",
      "              (self): BertSelfAttention(\n",
      "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "              (output): BertSelfOutput(\n",
      "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (intermediate): BertIntermediate(\n",
      "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
      "            )\n",
      "            (output): BertOutput(\n",
      "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
      "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (10): BertLayer(\n",
      "            (attention): BertAttention(\n",
      "              (self): BertSelfAttention(\n",
      "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "              (output): BertSelfOutput(\n",
      "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (intermediate): BertIntermediate(\n",
      "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
      "            )\n",
      "            (output): BertOutput(\n",
      "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
      "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (11): BertLayer(\n",
      "            (attention): BertAttention(\n",
      "              (self): BertSelfAttention(\n",
      "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "              (output): BertSelfOutput(\n",
      "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (intermediate): BertIntermediate(\n",
      "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
      "            )\n",
      "            (output): BertOutput(\n",
      "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
      "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "        )\n",
      "      )\n",
      "      (pooler): BertPooler(\n",
      "        (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "        (activation): Tanh()\n",
      "      )\n",
      "    )\n",
      "    (classifier): RobertaClassificationHead(\n",
      "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "      (dropout): Dropout(p=0.1, inplace=False)\n",
      "      (out_proj): Linear(in_features=768, out_features=6, bias=True)\n",
      "    )\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(learner.model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# For roberta-base\n",
    "list_layers = [learner.model.transformer.roberta.embeddings,\n",
    "              learner.model.transformer.roberta.encoder.layer[0],\n",
    "              learner.model.transformer.roberta.encoder.layer[1],\n",
    "              learner.model.transformer.roberta.encoder.layer[2],\n",
    "              learner.model.transformer.roberta.encoder.layer[3],\n",
    "              learner.model.transformer.roberta.encoder.layer[4],\n",
    "              learner.model.transformer.roberta.encoder.layer[5],\n",
    "              learner.model.transformer.roberta.encoder.layer[6],\n",
    "              learner.model.transformer.roberta.encoder.layer[7],\n",
    "              learner.model.transformer.roberta.encoder.layer[8],\n",
    "              learner.model.transformer.roberta.encoder.layer[9],\n",
    "              learner.model.transformer.roberta.encoder.layer[10],\n",
    "              learner.model.transformer.roberta.encoder.layer[11],\n",
    "              learner.model.transformer.roberta.pooler]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learner split in 14 groups\n",
      "[Sequential(\n",
      "  (0): Embedding(50265, 768, padding_idx=1)\n",
      "  (1): Embedding(514, 768, padding_idx=1)\n",
      "  (2): Embedding(1, 768)\n",
      "  (3): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (4): Dropout(p=0.1, inplace=False)\n",
      "), Sequential(\n",
      "  (0): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (1): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (2): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (3): Dropout(p=0.1, inplace=False)\n",
      "  (4): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (5): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (6): Dropout(p=0.1, inplace=False)\n",
      "  (7): Linear(in_features=768, out_features=3072, bias=True)\n",
      "  (8): Linear(in_features=3072, out_features=768, bias=True)\n",
      "  (9): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (10): Dropout(p=0.1, inplace=False)\n",
      "), Sequential(\n",
      "  (0): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (1): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (2): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (3): Dropout(p=0.1, inplace=False)\n",
      "  (4): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (5): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (6): Dropout(p=0.1, inplace=False)\n",
      "  (7): Linear(in_features=768, out_features=3072, bias=True)\n",
      "  (8): Linear(in_features=3072, out_features=768, bias=True)\n",
      "  (9): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (10): Dropout(p=0.1, inplace=False)\n",
      "), Sequential(\n",
      "  (0): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (1): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (2): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (3): Dropout(p=0.1, inplace=False)\n",
      "  (4): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (5): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (6): Dropout(p=0.1, inplace=False)\n",
      "  (7): Linear(in_features=768, out_features=3072, bias=True)\n",
      "  (8): Linear(in_features=3072, out_features=768, bias=True)\n",
      "  (9): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (10): Dropout(p=0.1, inplace=False)\n",
      "), Sequential(\n",
      "  (0): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (1): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (2): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (3): Dropout(p=0.1, inplace=False)\n",
      "  (4): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (5): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (6): Dropout(p=0.1, inplace=False)\n",
      "  (7): Linear(in_features=768, out_features=3072, bias=True)\n",
      "  (8): Linear(in_features=3072, out_features=768, bias=True)\n",
      "  (9): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (10): Dropout(p=0.1, inplace=False)\n",
      "), Sequential(\n",
      "  (0): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (1): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (2): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (3): Dropout(p=0.1, inplace=False)\n",
      "  (4): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (5): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (6): Dropout(p=0.1, inplace=False)\n",
      "  (7): Linear(in_features=768, out_features=3072, bias=True)\n",
      "  (8): Linear(in_features=3072, out_features=768, bias=True)\n",
      "  (9): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (10): Dropout(p=0.1, inplace=False)\n",
      "), Sequential(\n",
      "  (0): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (1): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (2): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (3): Dropout(p=0.1, inplace=False)\n",
      "  (4): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (5): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (6): Dropout(p=0.1, inplace=False)\n",
      "  (7): Linear(in_features=768, out_features=3072, bias=True)\n",
      "  (8): Linear(in_features=3072, out_features=768, bias=True)\n",
      "  (9): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (10): Dropout(p=0.1, inplace=False)\n",
      "), Sequential(\n",
      "  (0): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (1): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (2): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (3): Dropout(p=0.1, inplace=False)\n",
      "  (4): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (5): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (6): Dropout(p=0.1, inplace=False)\n",
      "  (7): Linear(in_features=768, out_features=3072, bias=True)\n",
      "  (8): Linear(in_features=3072, out_features=768, bias=True)\n",
      "  (9): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (10): Dropout(p=0.1, inplace=False)\n",
      "), Sequential(\n",
      "  (0): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (1): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (2): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (3): Dropout(p=0.1, inplace=False)\n",
      "  (4): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (5): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (6): Dropout(p=0.1, inplace=False)\n",
      "  (7): Linear(in_features=768, out_features=3072, bias=True)\n",
      "  (8): Linear(in_features=3072, out_features=768, bias=True)\n",
      "  (9): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (10): Dropout(p=0.1, inplace=False)\n",
      "), Sequential(\n",
      "  (0): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (1): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (2): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (3): Dropout(p=0.1, inplace=False)\n",
      "  (4): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (5): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (6): Dropout(p=0.1, inplace=False)\n",
      "  (7): Linear(in_features=768, out_features=3072, bias=True)\n",
      "  (8): Linear(in_features=3072, out_features=768, bias=True)\n",
      "  (9): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (10): Dropout(p=0.1, inplace=False)\n",
      "), Sequential(\n",
      "  (0): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (1): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (2): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (3): Dropout(p=0.1, inplace=False)\n",
      "  (4): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (5): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (6): Dropout(p=0.1, inplace=False)\n",
      "  (7): Linear(in_features=768, out_features=3072, bias=True)\n",
      "  (8): Linear(in_features=3072, out_features=768, bias=True)\n",
      "  (9): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (10): Dropout(p=0.1, inplace=False)\n",
      "), Sequential(\n",
      "  (0): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (1): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (2): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (3): Dropout(p=0.1, inplace=False)\n",
      "  (4): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (5): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (6): Dropout(p=0.1, inplace=False)\n",
      "  (7): Linear(in_features=768, out_features=3072, bias=True)\n",
      "  (8): Linear(in_features=3072, out_features=768, bias=True)\n",
      "  (9): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (10): Dropout(p=0.1, inplace=False)\n",
      "), Sequential(\n",
      "  (0): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (1): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (2): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (3): Dropout(p=0.1, inplace=False)\n",
      "  (4): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (5): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (6): Dropout(p=0.1, inplace=False)\n",
      "  (7): Linear(in_features=768, out_features=3072, bias=True)\n",
      "  (8): Linear(in_features=3072, out_features=768, bias=True)\n",
      "  (9): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (10): Dropout(p=0.1, inplace=False)\n",
      "), Sequential(\n",
      "  (0): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (1): Tanh()\n",
      "  (2): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (3): Dropout(p=0.1, inplace=False)\n",
      "  (4): Linear(in_features=768, out_features=6, bias=True)\n",
      ")]\n"
     ]
    }
   ],
   "source": [
    "learner.split(list_layers)\n",
    "num_groups = len(learner.layer_groups)\n",
    "print('Learner split in',num_groups,'groups')\n",
    "print(learner.layer_groups)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner.save('untrain')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "seed_all(seed)\n",
    "learner.load('untrain');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner.freeze_to(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CustomTransformerModel\n",
       "======================================================================\n",
       "Layer (type)         Output Shape         Param #    Trainable \n",
       "======================================================================\n",
       "Embedding            [512, 768]           38,603,520 False     \n",
       "______________________________________________________________________\n",
       "Embedding            [512, 768]           394,752    False     \n",
       "______________________________________________________________________\n",
       "Embedding            [512, 768]           768        False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [768]                590,592    True      \n",
       "______________________________________________________________________\n",
       "Tanh                 [768]                0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [768]                590,592    True      \n",
       "______________________________________________________________________\n",
       "Dropout              [768]                0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [6]                  4,614      True      \n",
       "______________________________________________________________________\n",
       "\n",
       "Total params: 125,240,838\n",
       "Total trainable params: 1,185,798\n",
       "Total non-trainable params: 124,055,040\n",
       "Optimized with 'transformers.optimization.AdamW', correct_bias=False\n",
       "Using true weight decay as discussed in https://www.fast.ai/2018/07/02/adam-weight-decay/ \n",
       "Loss function : FlattenedLoss\n",
       "======================================================================\n",
       "Callbacks functions applied \n",
       "    ShowGraph"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='' max='1' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.00% [0/1 00:00<00:00]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>error_rate</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='70' class='' max='575' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      12.17% [70/575 00:02<00:20 4.9422]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    }
   ],
   "source": [
    "learner.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Min numerical gradient: 3.02E-03\n",
      "Min loss divided by 10: 7.59E-08\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAEGCAYAAABLgMOSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXxV5bn3/8+ViQAZmMKcEEBlrICCU6titYitU63zVG2VWj2tHY61p0/PoefnaU9Pn1bbiq211qJV0VNrf2qr1lmcEZQ5iJBAiAEyQULm6Xr+2BsalJAQsrL2Tr7v12u/ZK/x2rc7+eZew73M3RERETlUCWEXICIi8UkBIiIiXaIAERGRLlGAiIhIlyhARESkS5LCLqA7DRs2zHNzc8MuQ0QkbqxYsaLM3bO6sm6vCpDc3FyWL18edhkiInHDzLZ2dV0dwhIRkS5RgIiISJcoQEREpEsUICIi0iUKEBER6RIFiIiIdIkCREREukQBIiISx15Yv5O7X91MGI/mUICIiMSxp1YX86e3tmJmPb5vBYiISBwrKKthQtbAUPatABERiVPuTn5pDROGKUBEROQQlO5poLqhmQlZaaHsXwEiIhKn8stqABivHoiIiByK/NJIgOgciIiIHJL80mr6JSUwOrN/KPtXgIiIxKmCshrGDxtIQkLPX8ILChARkbiVH+IlvKAAERGJS43NrRRW1DJhWDhXYEGAAWJm95lZiZmtbWf+YDP7q5mtNrNlZjY9Oj3bzF42szwzW2dmNwdVo4hIvNq2q5aWVg/tCiwItgeyGJh/kPk/AFa6+9HA1cCvotObge+6+xTgBOAmM5saYJ0iInEn7CuwIMAAcfelQMVBFpkKvBhddgOQa2Yj3H27u78Xnb4HyAPGBFWniEg8yi+tBuidh7A6YRVwAYCZHQeMA8a2XcDMcoFZwDvtbcTMFpjZcjNbXlpaGlixIiKxpKCshqEDU8gckBxaDWEGyE+BwWa2EvgG8D6Rw1cAmFka8BfgW+5e1d5G3P0ed5/t7rOzsrKCrllEJCbkl4Z7BRZAUlg7jobCtQAWGYe4IPrCzJKJhMdD7v54WDWKiMSq/LJqTp88ItQaQuuBmNkgM0uJvr0OWOruVdEw+QOQ5+63h1WfiEisqqxroqy6kfG9tQdiZkuAucAwMysCFgLJAO5+NzAFeMDMWoD1wFejq34auApYEz28BfADd386qFpFROJJQXQQxbCGcd8rsABx98s6mP8WcOQBpr8OhHNfvohIHNh3BVZIw7jvpTvRRUTiTH5pDYkJRs6QAaHWoQAREYkzBWU1ZA/uT0pSuL/CFSAiInFmc2l16IevQAEiIhJXWludLeU1oY6BtZcCREQkjmyvqqe+qTX0mwhBASIiEldiYQysvRQgIiJxZO8ovBPVAxERkUNRUFbDwJREstL7hV2KAkREJJ7svQIrMupTuBQgIiJxJBZG4d1LASIiEifqm1oorqyLiUt4QQEiIhI3tpTX4B7+GFh7KUBEROLEvuegqwciIiKHYu8w7jqEJSIih2RzaTUjM1IZ2C+0h8nuRwEiIhInYukKLAgwQMzsPjMrMbO17cwfbGZ/NbPVZrbMzKa3mTffzD4ws01m9v2gahQRiRfuTn5pdcwcvoJgeyCLgfkHmf8DYKW7Hw1cDfwKwMwSgbuAs4CpwGVmNjXAOkVEYl5FTSNV9c0xcwUWBBgg7r4UqDjIIlOBF6PLbgByzWwEcBywyd3z3b0ReAQ4L6g6RUTiQf7e56D3hUNYnbAKuADAzI4DxgFjgTHAtjbLFUWnHZCZLTCz5Wa2vLS0NMByRUTCUxBjl/BCuAHyU2Cwma0EvgG8DzQDBxrgxdvbiLvf4+6z3X12VlZWMJWKiIQsv6yGlMQExg4O9znobYV2LZi7VwHXAlhkVLCC6GsAkN1m0bFAcY8XKCISQ7aW1zB2SH8SE8IfRHGv0HogZjbIzFKib68DlkZD5V3gSDMbH51/KfBkWHWKiMSCwopaxg2Jnd4HBNgDMbMlwFxgmJkVAQuBZAB3vxuYAjxgZi3AeuCr0XnNZvYvwD+AROA+d18XVJ0iIrHO3Sksr2X2uMFhl7KfwALE3S/rYP5bwJHtzHsaeDqIukRE4s3u2ib2NDSTHWM9EN2JLiIS4woragEYNzR2rsACBYiISMzbGg2QHPVARETkUGxTgIiISFdsLa8hK70f/VMSwy5lPwoQEZEYV1hRG3O9D1CAiIjEvG0VdTF3DwgoQEREYlpDcwvFlXUxdwkvKEBERGLaR7vqcI+9E+igABERiWn/vAdEASIiIoegMEYv4QUFiIhITCssryU1OYGs9H5hl/IJChARkRi2NXoJb+SpF7FFASIiEsO2xeg9IKAAERGJWe4evYkwtgZR3EsBIiISo8qqG6ltbCFnSP+wSzmgwALEzO4zsxIzW9vO/Ewze8rMVpnZOjO7ts28b0enrTWzJWaWGlSdIiKxKlaHcd8ryB7IYmD+QebfBKx39xlEnlz4CzNLMbMxwDeB2e4+nchTCS8NsE4RkZhUWFEDEJN3oUOAAeLuS4GKgy0CpFvk0oK06LLN0XlJQH8zSwIGAMVB1SkiEqsKy+swg7GD+9ghrE5YROS56MXAGuBmd29194+AnwOFwHag0t2fa28jZrbAzJab2fLS0tKeqFtEpEcUVtQyMiOV1OTYGsZ9rzAD5ExgJTAamAksMrMMMxsMnAeMj84baGZXtrcRd7/H3We7++ysrKyeqFtEpEcUVtTE7OErCDdArgUe94hNQAEwGTgDKHD3UndvAh4HTgqxThGRUMTqc0D2CjNACoHTAcxsBDAJyI9OP8HMBkTPj5wO5IVWpYhICOqbWthZ1RCTzwHZKymoDZvZEiJXVw0zsyJgIZAM4O53A7cBi81sDWDAre5eBpSZ2WPAe0ROqr8P3BNUnSIisWjfc9BjcBTevQILEHe/rIP5xcC8duYtJBI4IiJ9UiyPwruX7kQXEYlBW8sVICIi0gWFFbUMTElkyMCUsEtplwJERCQGbauoJWfowJgcxn0vBYiISAyKPAckNu9A30sBIiISY1pbnW0VtTE7iOJeChARkRhTsqeBhubWmL4LHRQgIiIxJx4u4QUFiIhIzNn3HBAFiIiIHIrC8hoSDEYP0kl0ERE5BIUVtYwe1J+UpNj+FR3b1YmI9EGxPgrvXgoQEZEYowAREZFDVtPQTFl1Y0yPwruXAkREJIbEyyW8oAAREYkpChDAzO4zsxIzW9vO/Ewze8rMVpnZOjO7ts28QWb2mJltMLM8MzsxqDpFRGKFu/Ne4S4Axg2J7WFMIMAHSgGLgUXAA+3MvwlY7+7nmFkW8IGZPeTujcCvgGfd/UIzSwFiP4pFRLrI3Xl1Yyl3PL+RVUWVzB43mMwByWGX1aHAeiDuvhSoONgiQHr0uedp0WWbzSwDOAX4Q3Q7je6+O6g6RUQOy+bNcOONkJEBCQmR/954Y2R6J7y5uYyL7n6La/74LmXVjfzswqN5ZMEJARfdPYLsgXRkEfAkUAykA5e4e6uZTQBKgT+a2QxgBXCzu9ccaCNmtgBYAJCTk9MjhYuIAPDMM3DhhdDUFHkB7NkD994L998Pjz0GZ511wFULy2v5/uOreXNzOSMzUvmv86dz8ezsmL95sK0wKz0TWAmMBmYCi6K9jyTgGOC37j4LqAG+395G3P0ed5/t7rOzsrJ6oGwRESI9jAsvhNraf4bHXk1NkekXXthuT2Thk2tZXVTJf5w9lVdumcuVJ4yLq/CAcAPkWuBxj9gEFACTgSKgyN3fiS73GJFAERGJHb/4xSeD4+OamuCOOz4xeV1xJS9/UMoNp07gK58ZT2pyYkBFBivMACkETgcwsxHAJCDf3XcA28xsUnS504H14ZQoIrK/wvJabn1sNfWLH+hcgPzpT5+Y/NtXNpPWL4mrTswNpsgeEtg5EDNbAswFhplZEbAQSAZw97uB24DFZrYGMOBWdy+Lrv4N4KHoFVj5RHorIiKhW/Tyhzz+3kf8d11t51aort7vbUFZDU+v2c6CUyaS2T/2r7Q6mE4FiJkNBOqiJ7mPInKo6Rl3bzd+3f2yg23T3YuBee3MWwnM7kxtIiI9pbKuiSdXFXPR7GwS0tMiJ8w7kpa239vfvbqZpMQEvvKZ3GCK7EGdPYS1FEg1szHAi0R6BIuDKkpEJBb99b0i6ptaueL4HLjySkjuoAeRnAxXXbXv7Y7Kev7yXhEXzx7L8PTUgKsNXmcDxNy9FrgAuNPdvwhMDa4sEZHY4u489E4hM7IHMX1MJnz3u50LkG9/e9/be1/Lp9Xha6dMDLjantHpAIkOJ3IF8PfotDDvIRER6VHvbtnFhyXVXHFc9H6ziRMj93kMGPCJIGlKSKQ+OZWmR/83shywq6aRh5cVcu6M0WTHwThXndHZAPkW8G/AX919XfRmv5eDK0tEJLY89M5W0lOTOHvGqH9OPOssWL0aFizY7070HZdczbxr7+SegZP2Lbr4zS3UNrbw9bm9o/cBnexFuPurwKsAZpYAlLn7N4MsTEQkVpRXN/DMmh1cfnwOA1I+9mtz4kRYtCjyisoGpv5pBb9+8UPOOXo0Q9NSWPzmFj43dQRHjUjv2eID1KkeiJk9bGYZ0aux1hMZ+PCWYEsTEYkNj60oorGllcuP7/xwST86dxrJiQn88Im1PPxOIZV1TdzYi3of0PlDWFPdvQo4H3gayAGuOvgqIiLxr7XVeXhZIcflDjmk3sPIzFT+dd5RLN1Yys+f+4CTJg5lVs7gACvteZ0NkGQzSyYSIE9E7//w4MoSEYkNb2wuY2t5LVeccOiDtV51Yi5Hj82kobmVG+ceEUB14erslVS/A7YAq4ClZjYOqAqqKBGRWPHQ24UMGZjC/OkjD3ndxATjrsuP4Y1NZXz6iKEBVBeuTvVA3P3X7j7G3T8fHfxwK3BawLWJiIRqZ1U9z+ft5KJjx9IvqWsDHmYPGcClx+UQefRR79LZk+iZZna7mS2Pvn4BxP7zFkVEDsOj726jpdW57Dg9a+hAOnsO5D5gD3Bx9FUF/DGookREwtbc0sqSZYWcfOQwcofp7+UD6ew5kInu/qU27//TzFYGUZCISCx4eu0OtlfWs/AcjdrUns72QOrM7DN735jZp4G6YEoSEQlXSVU9C59Yy/QxGZwxZUTY5cSszvZAbgAeMLPM6PtdwJeDKUlEJDzuzi2PraauqYVfXjKLpMT4esxsT+rsUCargBnRZ5bj7lVm9i1gdZDFiYj0tD+9vZVXN5Zy23nTOGJ4Wscr9GGHFK3uXhW9Ix3gOwdb1szuM7MSM1vbzvxMM3vKzFaZ2Tozu/Zj8xPN7H0z+9uh1Cgi0lWbSvbw47/nMXdSFleeMC7scmLe4fTNOrqoeTEw/yDzbwLWu/sMIo++/UX0EbZ73QzkHUZ9IiKd1tjcys2PrGRgvyR+duHRvfK+je52OAFy0KFM3H0pUNHB+ukW+b+UFl22GcDMxgJfAO49jPpERDrtly9sZF1xFT+94FO94mmBPeGg50DMbA8HDgoD+h/mvhcBTwLFQDpwibu3Ruf9EvhedPpBmdkCYAFATo5u9hGRQ7esoILfvrqZS+dkM2/aoQ9Z0lcdtAfi7ununnGAV7q7H+4TCc8EVgKjgZnAouiQ8WcDJe6+ojMbcfd73H22u8/Oyso6zJJEpK+pqm/i24+uJGfIAP79bN3zcSjCvD7tWuDx6Nham4ACYDLwaeBcM9sCPAJ81sweDK9MEenNbn9uI9sr67jjkpkM7KcndR+KMAOkEDgdwMxGAJOAfHf/N3cf6+65wKXAS+5+ZXhlikhvlbe9igfe2sIVx4/jmF72rI6eEFjcmtkSIldXDTOzImAhkAzg7ncDtwGLzWwNkXMqt7p7WVD1iIi05e4sfHIdmf2T+e68o8IuJy4FFiDuflkH84uBeR0s8wrwSvdVJSIS8eSqYpYVVPDfF3yKQQNSOl5BPkH36ItIn1Pd0MxPns7j6LGZXDw7O+xy4pbOGIlIn3Pnix+ys6qBu688lsQE3TDYVeqBiEifsqmkmj+8XsDFs8cySyfOD4sCRET6DHfnR0+uo39KIt+bPznscuKeAkRE+ox/rNvB65vK+O7njmJYWr+wy4l7ChAR6RPqGlu47W95TB6ZrpF2u4lOootIn/DQO1v5aHcdjyw4QQ+J6iZqRRHp9RqbW7n3tQJOmDCEEyYMDbucXkMBIiK93hMrP2JHVT03nDox7FJ6FQWIiPRqra3O75bmM2VUBqcepRG7u5MCRER6tRc3lLCppJobTp2gpwx2MwWIiPRqd7+6mbGD+/OFT40Ku5ReRwEiIr3Wu1sqWLF1F9efPEFXXgVALSoivdbdr2xmyMAUDZgYEAWIiPRKH+zYw4sbSvjyibn0T0kMu5xeKbAAMbP7zKzEzNa2Mz/TzJ4ys1Vmts7Mro1Ozzazl80sLzr95qBqFJHe63dLN9M/OZGrT9Rd50EJsgeyGJh/kPk3AevdfQaRJxf+wsxSgGbgu+4+BTgBuMnM9KR7Eem0j3bX8eTKYi49LpvBA/WwqKAEFiDuvhSoONgiQLpFrqtLiy7b7O7b3f296Db2AHnAmKDqFJHe5w+vFQBw3ckTQq6kdwvzHMgiYApQDKwBbnb31rYLmFkuMAt4p72NmNkCM1tuZstLS0uDq1ZE4sLu2kYeebeQc2eMZsyg/mGX06uFGSBnAiuB0cBMYJGZZeydaWZpwF+Ab7l7VXsbcfd73H22u8/OytJdpiJ93TNrd1Db2MJXPjM+7FJ6vTAD5FrgcY/YBBQAkwHMLJlIeDzk7o+HWKOIxJkX83YydnB/po3O6HhhOSxhBkghcDqAmY0AJgH50XMifwDy3P32EOsTkThT39TC65vKOH3ycA1b0gMCex6ImS0hcnXVMDMrAhYCyQDufjdwG7DYzNYABtzq7mVm9hngKmCNma2Mbu4H7v50ULWKSO/wxqYy6ptaOX3KiLBL6RMCCxB3v6yD+cXAvANMf51IoIiIHJIXN5QwMCWR4ycMCbuUPkF3ootIr+DuvJRXwslHZtEvSXee9wQFiIj0CuuKq9hRVc/pU4aHXUqfoQARkV7hxbwSzOC0yQqQnqIAEZFe4cUNO5mZPYhhaf3CLqXPUICISNzbWVXP6qJKztDVVz1KASIice/lDSUAOv/RwxQgIhL3XsgrYcyg/kwakR52KX2KAkRE4lrk7vNSTp+iu897mgJEROLaW5vLdfd5SBQgIhLXXsjbyYCURI4fr7vPe5oCRETilrvz0oYSTj5yGKnJuvu8pylARCRurd9exfbKeh2+CokCRER6nLt3y3b23X0+SZfvhkEBIiI9aktZDbNue56FT6ylobnlsLb1Yt5OZowdRFa67j4PgwJERHrUva/nU1XXxP1vbeWiu99iW0XtIa1fUdPIG5vKuGfpZlYVVXKGbh4MTZAPlLoPOBsocffpB5ifCTwI5ETr+Lm7/zE6bz7wKyARuNfdfxpUnSLScypqGvnz8iIuOjab0yYP55bHVvGFX7/GLy6eyeemfvI8Rmur8/623by0YSfriqvI217FzqqGffNHZ6Zy9tGje/IjSBuBBQiwGFgEPNDO/JuA9e5+jpllAR+Y2UNAC3AX8DmgCHjXzJ509/UB1ioiPeDBt7fS0NzKdSeP58gR6UwdlcGND6/g+geWs+CUCdxy5iQSzXh/227+vno7z6zdzvbKepISjCOGp/HpicOYMiqDKaMymDwqXQMnhizIJxIuNbPcgy0CpEefgZ4GVADNwPHAJnfPBzCzR4DzAAWISByrb2rh/je3cNqkLI6MDjmSM3QAj91wEj/+ex73LM1n6cZSKuua2F5ZT0piAqccNYxbzpzEGVNHkJGaHPInkI8LsgfSkUXAk0AxkA5c4u6tZjYG2NZmuSIioSIiceyv739EeU0j158yYb/pqcmJ3Hb+dI4bP4SfP/cB00Zn8r35kzh9ikIj1oUZIGcCK4HPAhOB583sNQ78PPR2r/kzswXAAoCcnJwAyhSRw9Xa6vz+tXymj8ngxAlDD7jMOTNGc84Mnc+IJ2FehXUt8LhHbAIKgMlEehzZbZYbS6SXckDufo+7z3b32VlZWYEWLCJd8/IHJeSX1nD9yRM04GEvEmaAFAKnA5jZCGASkA+8CxxpZuPNLAW4lMihLhGJU/cszWfMoP58/lOjwi5FulGQl/EuAeYCw8ysCFgIJAO4+93AbcBiM1tD5LDVre5eFl33X4B/ELmM9z53XxdUnSISrFXbdvNOQQU//MIUkhN161lvEuRVWJd1ML8YmNfOvKeBp4OoS0R61u9fyye9XxKXzMnueGGJK/pzQEQCs62ilmfW7uDy43NI1xVVvY4CREQC88c3tmDANZ/ODbsUCYACREQCsa2ilkffLeTcGaMZldk/7HIkAAoQEel25dUNfPm+ZSQmGN84/ciwy5GAhHkjoYj0QrWNzXzl/uV8tLuOh647nvHDBoZdkgREPRAR6TZNLa3c+NB7rCnazZ2XzWJ2rp5T3pupByIi3cLd+f5f1vDKB6X85IufYt60kWGXJAFTD0REusXP/vEBf3mviG+dcSSXH69x6foC9UBE5LC0tjr3vVHAb1/ZzOXH53CzTpr3GQoQEdmPu/P3Ndupb2pl8sh0jhyRRr+kxP2WqWts4Y1NZbyQt5MX8kooq25g3tQR3HbedA2W2IcoQERkP/e+VsCPn87b9z4pwZiYlcbkUelMzEpjdVElr28qpb6plfR+SZw6KYvPTR3BWdNHkZig8OhLFCAiss+LeTv5yTN5fP5TI/nO5yaRt72KDTuqyNu+h2UFFTyxspgxg/pzyexszpg6guPHDyUlSadS+yoFiIgAkLe9im8ueZ/pozP5xUUz6Z+SyBHD0/Z7yFN1QzMDUxJ1mEoAXYUlEhOaW1ppbW33wZuBK93TwHX3LyctNYnfXz2b/imJB1wurV+SwkP2UYCIhGxLWQ2n3/4qZ9/5Ovml1T2+//qmFm54cAXlNQ3ce/UcRmam9ngNEp8UICIhWlNUyZd++yZ76pvZXlnHOXe+zhMrP+qx/bs7P3h8DSu27uL2i2fyqbGZPbZviX9BPpHwPuBsoMTdpx9g/i3AFW3qmAJkuXuFmX0buA5wYA1wrbvXB1WrxI9lBRW8vqmMMYNSyRkykHFDBzAyI5WEOLz65/UPy/jan5YzaEAKf/rqcaQmJ/LNJe9z8yMreTu/goXnTCU1+cCHkrpDa6tz50ubePz9j/ju547S42blkJl7MMddzewUoBp44EAB8rFlzwG+7e6fNbMxwOvAVHevM7P/BZ5298Ud7XP27Nm+fPnybqheYk1ZdQM/+Xsej7//yb/OU5ISyB7cn3FDI4EybsgAxg0bSO7QgYwd3D8mH6P6t9XFfPvRlUzMSuP+rxzHiIzIYaOmllZuf34jv31lM5NHpnPXFccwMSutW/fd2ur8Y90OfvXih2zYsYfzZ47mjktm6txGH2VmK9x9dlfWDfKRtkvNLLeTi18GLGnzPgnob2ZNwACguHurk3jR0uo8vKyQ//vsBuqaWviX047ghrkTqahuZGtFDYUVtRSW17K1vJatFbW8k19OTWPLvvUTDI4akc4XZ43hi7PGMDwj/OP797+5hR89tY7Z4wZz75fnkNn/n0/qS05M4Nb5kzlu/BC+8+hKzrnzdf7j7KlcPDu7w17Wzqp6Fr20iVZ3jskZzDHjBpM7dMC+YPh4cEzIGsivLp3J2UePVnhIlwTWAwGIBsjfDtYDMbMBQBFwhLtXRKfdDPwYqAOec/crDrL+AmABQE5OzrFbt27ttvolXGs/quT//HUNq4oqOXHCUG47fzpHDD/4X+PuTll1I1vLa9haXsuW8hre3FzOiq27SDA4+cgsvnTsWOZNHRHo4aEDaWhu4RfPbeSepfmcMWUEiy6fddAatlfW8a1HVvJOQQVHj81k4TnTOHbc4E8s19TSyuI3tvDLFzbS1Or0S0xgT0MzAEMGpjArexDTRmfw3Pqd+4Lj5tOP5OyjR+vGPzmsHkgsBMglwJXufk70/WDgL8AlwG7gz8Bj7v5gR/vTIaz45+68lV/Ow+8U8vSa7QwZ2I9/P3sK5844vL+S80urefy9j3j8vSKKK+tJT03igllj+PrcI3rkqqP3C3fxvcdW82FJNVccn8N/njuNpE4cWnN3nlhZzH8/k8fOqgYumDWGW8+avO+Q15uby1j4xDo+LKnmtElZ/OjcaYwdPIBNJdW8V7iL97bu4r3CXWwurVFwyAHFe4D8Ffizuz8cfX8RMN/dvxp9fzVwgrvf2NH+YjVAqhuaqWtswQwSzEgwMIzkJGNAiu7lBNhd28hjK4p4eFkh+aU1ZPZP5pI52dx02hH7HeI5XK2tkYB6bEURT60qJiHBuOqEcXx97kSGpfXrtv3sVdfYwh0vbOTe1/IZkZHKTy74FKdNGn7I26lpaOY3r2zi90sLSEo0bpw7kY07q3lyVTFjB/dn4TnTOGPK8HZDtqahmf7JiXF5sYEEK24DxMwygQIg291rotOOB+4D5hA5hLUYWO7ud3a0v1gLkJ1V9dz50oc8+u42mloO3M5XnziOH35hap8dDqKgrIY7X/qQv63eTmNzK8eOG8zlx+XwhaNHBX6IaVtFLb968UMef6+I1ORErjkpl6+dMpHMAd0TWO/kl3PrX1azpbyWy4/P4d/Omkx66uFte2t5Df/19zyeX7+TlKQEbjh1IjfOndjjh+Ok94jJADGzJcBcYBiwE1gIJAO4+93RZa4h0tu49GPr/ieRQ1jNwPvAde7e0NE+YyVAdtU0cverm1n85hZaWp2L52QzZVQGuNPq0OqOO3xYUs2SZYXMyhnEb644hlGZ/cMuvce4O0uWbeO2v60nMcH44qwxXH58TqSdetjm0mp++cKHPLWqmPTUJC6dk81JE4dxbO5gMjr5C7+qvonNJdVsLq1hc2k1H+zYw0sbSsge0p//ueBoTjpiWLfWvPajSgYNSGbs4AHdul3pe2IyQMIQdoBUNzTzh9cKuPe1fKobm/nizDF864yjyBna/g/502u2c8ufV9E/JZE7LwZIVKQAAAwBSURBVDuGEycO7cGKw1FW3cD3/7KaF/JKOPnIYfzfC2fExN3PeduruOP5jby0oYTmVscMpozM4LjxQzhu/BBGZPRjR2UD2yvr2FFZz46qenZU1rO1opbSPf/8+yYpwcgdNpDPTh7Ot844UocpJaYpQKJ6OkAamltYU1TJsi0VLCuoYPmWXVQ3NDN/2ki+M+8ojhqR3qntbCrZw9f+tIIt5bXcOn8S1588IbDLKnfVNPLU6mLS+iUxMiOVkZmRV0/9kntpw06+99hqquqb+f78yVxzUm7MHZevbWxmZeFu3imo4N0tFbxXuIv6ptb9lklNTtjXfmMHD+CI4WlMzEpjYtZAsocMiMl7T0QORAES1RMB0tLq/OH1fF7MK2Hltt00NEd+sRwxPI05uUO4dE42M7IHHfJ2qxuaueXPq3hm7Q7OnDaCOblDqG5oZk99M9X1zVQ3NNPQ3MoZU4Zz3swx7Q521x5358lVxfx/T62nvKbxE/MzUpMYPag/U0ZlMG10BtNGZzJ1dEa3ncAur27gjhc28uDbhUwemc6vLp3FpJGdC9iwNTa3sra4ksraJkZmpjIqM5XM/sm6d0J6BQVIVNAB0tTSyr/+eRVPrCxm+pgMjh8/lDm5Q5iTO5ih3XAFj7vz+9fy+Z9nP6AlOjLrwJRE0lKTSOuXRFOLU1hRS0ZqEpfMyebKE8YxbujADrf70e46/v3/X8tLG0qYkT2I/zpvOmmpSWyvrGNnVT3bKyOHYrZV1LJ+exU7q/55OCZnyABmZg/ia6dOYNroQxsnaWt5Dc+t28nz63eyfGsFDlx/8gS+O++oTzzhTkTCoQCJCjJAGppb+MbD7/Pc+p3cOn8yX587MZD9AOypb8KBgSlJ+12v7+4s37qL+9/cwrNrd9DizmmThnPF8TlMHpXBsLSU/X4xt7Q6D769lZ89u4FWh1vOnMSXT8rt8B6A0j0NrCuuZF1xFeuKK3ljUzlV9U18ceYYvjPvqHZP3Lo767dX8ezaHTy3bicf7NwDwOSR6cybNpLPf2okk0f2/ElyEWmfAiSqKwHi7nxjyfvMHjeYS+bkHPDQUF1jC197cAVLN5byo3Omcs2nx3dXyV22s6qeh98p5OFlhfudwM3sn8zw9H5kpfdjd20T67dXccpRWfz4/OlkD+naFTuVdU389pXN3PdGAQDXnpTLjXOP2He56wc79vC31cX8ffV28stqSEww5uQOZt7UkXxu6ogu71dEgqcAiepKgFTVN3Hd4uUs21LB0IEpfOUz47nqxHH7Lt+sbmjmK4vf5d0tFfzPBUdz8ZzsIErvssbmVt7OL6d4dx2lexoorW6I/HdPA7WNLVx/ynjOnzmmW47Xf7S7jtuf28jj7xeRkZrMuTNG83Z+OR+WVJNgcMKEoZx99GjmTx/JkIEp3fDpRCRoCpCowzmEtayggrte3sSrG0tJ75fE1SeN40vHjOXb/7uKtR9VcvvFMzhv5phurjg+rS+u4qfPbuC1D0uZkzuEc44exfzpo8hK7/47uUUkWAqQqO44B7L2o0p+88omnlm7A3dISUxg0eWzmDdtZDdV2Xs0Nrf22TvoRXqLmBzOPV5NH5PJb644lk0l1Tz49lbmTR3R7XcR9xYKD5G+TQHSjiOGp/Gjc6eFXYaISMzSn5AiItIlChAREekSBYiIiHSJAkRERLpEASIiIl2iABERkS5RgIiISJcoQEREpEt61VAmZlYJfNjO7EygspPTPz6to/fDgLJDKrZz2qu5O9Y52HJBtVVQ7dRebd2xTlDt9PFpsfyd6ux6YXynILba6nDb6WDzg/r5G+fuWQcruF3u3mtewD2HOu9A0z8+rRPvl/f05zncdcJoq6DaKci2CqqdDtA2MfudCrutetvPX0fLxNPPX287hPVUF+YdaPrHp3X0Pihd2U9n11FbdW6doNrp49NiuZ06u56+U4ffTgebH3Nt1asOYYXFzJZ7F0ez7EvUTp2ntuo8tVXnBNFOva0HEpZ7wi4gTqidOk9t1Xlqq87p9nZSD0RERLpEPRAREekSBYiIiHSJAqQNM7vPzErMbG0X1j3WzNaY2SYz+7WZWZt5F5vZejNbZ2YPd2/V4QiirczsGjMrNbOV0dd13V95zwvqexWdf6GZuZnF/UnkgL5TN0SnrzSz181savdX3vMCaqvvRH9PrTazF81sXIcbC+L66Xh9AacAxwBru7DuMuBEwIBngLOi048E3gcGR98PD/tzxnBbXQMsCvuzxUNbReelA0uBt4HZYX/OWGwnIKPNMucCz4b9OWO4rU4DBkT//XXg0Y62pR5IG+6+FKhoO83MJprZs2a2wsxeM7PJH1/PzEYR+aK+5ZHWfwA4Pzr7euAud98V3UdJsJ+iZwTUVr1SgG11G/AzoD7A8ntMEO3k7lVtFh0I9IqrhgJqq5fdvTa66NvA2I7qUIB07B7gG+5+LPCvwG8OsMwYoKjN+6LoNICjgKPM7A0ze9vM5gdabbgOt60AvhTtQj9mZtnBlRq6w2orM5sFZLv734IuNGSH/Z0ys5vMbDORsP1mgLWGrTt+/vb6KpHeyUEldaHIPsPM0oCTgD+3OfTc70CLHmDa3r90kogcxppLJNFfM7Pp7r67e6sNVze11VPAEndvMLMbgPuBz3Z3rWE73LYyswTgDiKH/HqtbvpO4e53AXeZ2eXAD4Evd3Opoeuutopu60pgNnBqR/tVgBxcArDb3We2nWhmicCK6Nsngd+yf3dvLFAc/XcR8La7NwEFZvYBkUB5N8jCQ3DYbeXu5W2m/x74n8CqDdfhtlU6MB14JfrLYiTwpJmd6+7LA669J3XHz19bj0SX7Y26pa3M7Azg/wCnuntDh3sN+2RQrL2AXNqcmALeBC6K/tuAGe2s9y5wAv88MfX56PT5wP3Rfw8DtgFDw/6cMdpWo9os80UiwRv654zFtvrYMq/QC06iB/SdOrLNMucQ4ICevaCtZgGb27ZZhzWE3Qix9AKWANuBJiI9h68C44FngVXAeuA/2ll3NrA2+j9gEf+8y9+A26PrrgEuDftzxnBb/TewLrr+y8DksD9nrLbVx5bpFQES0HfqV9Hv1Mrod2pa2J8zhtvqBWBntK1WAk92VIeGMhERkS7RVVgiItIlChAREekSBYiIiHSJAkRERLpEASIiIl2iAJFezcyqe3h/b3bTduaaWaWZvW9mG8zs551Y5/zeMtqsxAcFiMghMLODjt7g7id14+5ec/dZRG7wOtvMPt3B8ucDChDpMRrKRPocM5sI3AVkAbXA9e6+wczOITJWUgpQDlzh7jvN7EfAaCJ3/paZ2UYgB5gQ/e8v3f3X0W1Xu3uamc0FfgSUERl2ZAVwpbu7mX2eyM2lZcB7wAR3P7u9et29zsxW8s+BFK8HFkTr3ARcBcwkMlz5qWb2Q+BL0dU/8TkPo+lE9qMeiPRF7Y1a+jpwQvSv/keA77VZ51jgPHe/PPp+MnAmcByw0MySD7CfWcC3iPQKJgCfNrNU4HdEnsHwGSK/3A/KzAYTGT9taXTS4+4+x91nAHnAV939TSJjHd3i7jPdffNBPqdIt1APRPqUDkYtHQs8Gn1mQgpQ0GbVJ929rs37v3tksLkGMysBRrD/MNkAy9y9KLrflUR6MNVAvrvv3fYSIr2JAznZzFYDk4CfuvuO6PTpZvZfwCAgDfjHIX5OkW6hAJG+5oCjlkbdCdzu7k+2OQS1V83Hlm07UmkLB/5ZOtAyBxpOuz2vufvZZnYU8LqZ/dXdVwKLgfPdfZWZXUPkUQEfd7DPKdItdAhL+hSPPKGuwMwuArCIGdHZmcBH0X8H9cyIDcAEM8uNvr+koxXcfSORgSZvjU5KB7ZHD5td0WbRPdF5HX1OkW6hAJHeboCZFbV5fYfIL92vmtkqIiO1nhdd9kdEDvm8RuQEd7eLHga7EXjWzF4nMvppZSdWvRs4xczGA/8OvAM8TySQ9noEuCV66e9E2v+cIt1Co/GK9DAzS3P3aoucnLgL+NDd7wi7LpFDpR6ISM+7PnpSfR2Rw2a/C7kekS5RD0RERLpEPRAREekSBYiIiHSJAkRERLpEASIiIl2iABERkS75f+0DuYG9NPvrAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.recorder.plot(skip_end=10,suggestion=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>error_rate</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.735630</td>\n",
       "      <td>1.730995</td>\n",
       "      <td>0.213099</td>\n",
       "      <td>0.786901</td>\n",
       "      <td>00:16</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXxV1b338c8vJ/MESUggJISAgEwyaEQsFlGrghN6W3ux1bbeVqq91tbep0/t7b3V23rb29aXz1U7UG6lo8VOekXFqQ6gRYSgzPMQJARIICQkkDm/549zoAEDOUhCwub7fr3OK2evtfY5ayn5sll777XN3RERkeCK6e4OiIhI11LQi4gEnIJeRCTgFPQiIgGnoBcRCbjY7u5Ae/r06eOFhYXd3Q2JqDzYyM6qOgBCMcbAzGRSEnrkHx2Rs9ayZcv2unt2e3U98re1sLCQ4uLi7u6GANv3HeTmWW/TWtPA6Lx0dlfXs7e2kWEDejMoK5nS/XX0To4jOy2Rm4vyGZPXi9iQ/qEocrqZ2fbj1fXIoJee48evbWbfwUbm3jGRi8/JoupQIz9bsIWfL9jKih1VFA3MoKyqnrc272XukvcZkpNKZnI8X7rsHBLjQry5qYKrR/UjLhRDVko86UlxJMaFuntYImcV64k3TBUVFbmO6LtffVMLRQ/+lWmj+/Gjm8ceVffu+/upb2zhI0P6ALC27ADffW4tb2/d1+Hn5vVO4sGbRnPp0GxiYqxL+i5ytjGzZe5e1F6djujluF5dV05tQzM3js/7QN35BRlHbY/sn87cmROpqGngUGMzL6zezd827+WqUf1oaGrh7S37GNI3lXe2VrJ8RxW3/3IpN43PY+OeGsbk9+KuS4dQkJV8uoYmclbREb18wJNL3ueHL22g8mAjOWkJvP3NKwh10pF3a6uzsbyGax55k9Y2f/SG90tj3t2XEB+r+X05eU1NTZSWllJfX9/dXelyiYmJ5OfnExcXd1S5juglarUNzdw/bw0Nza0AXDWqb6eFPEBMjDG8XzrPfvkSrn30LQBm33YBM3+7jDt+U8xjnxrPrqp6+qUn0is5roNPEwkrLS0lLS2NwsJCzII7Heju7Nu3j9LSUgYNGhT1fgp6OaK11fnP59fS0NzKY7eM55nlO7njo4O75LtG9e/F5z5SSCjGuGpUP75yxVAeeXUTYx54+UibYX1T+WTRAL7QRX2Q4Kivrw98yAOYGVlZWVRUVJzUfgp6AaChuYUH5q1h7pIdDO+XxjXn5XL92P5d+p0P3DDqyPt7rxzGjspDvLRmNwcbWwCob2rlwefXkZUaz03j87u0L3LmC3rIH/ZhxqmgP8u1tjq/Xbyd51fuYklJJV+4ZBDfunZEt/zSPHTzWH74iTHEhmJoaG4hxoxP/+Id7v3DCr43fz3nZKdw2bk5mMGEQVmMze911vxyi5wKnfk6y/1swRbun7eGJSWV3HnpOfzbdSO7LTxjYuzIzVYJsSHiQjH8z21FfHPacKYMy2ZXdT3ff2E935u/nht/8jdmLdjaLf0UOVZVVRU//elPT3q/a665hqqqqi7o0dF0RH+Wqm9q4T+fX8dvF4dvpivITOaLk3veXHiv5Di+eOk5ADS1tPJ+5SFq6pv5+p9W8IMX1/Py2t18/pJBZKcmsHxHFSP7p3N+QYaWaJDT6nDQf+lLXzqqvKWlhVDo+DcIzp8/v6u7Bijoz0o7Kg/x/RfWMX/VbnLSEvjFZ4sYk9+7u7vVobhQDOdkpwLwpzsv5rbHl7C14iB3//69o9r175XIH754MQMydV2+nB733XcfW7ZsYdy4ccTFxZGamkpubi7Lly9n7dq13HjjjezYsYP6+nq+8pWvMHPmTODvy73U1tYybdo0LrnkEhYtWkReXh7PPPMMSUlJndI/XUd/lnl5zW7u/N0yWh0uOzebX94+obu7dEpaWp1X1+3hYGMzRQMzeXH1bh57bRMH6pu5dFg237p2BMP6pnV3N6WLrVu3jhEjRgDwH8+uYW3ZgU79/JH907n/+lHHrS8pKeG6665j9erVvPHGG1x77bWsXr36yCWQlZWVZGZmUldXx4UXXsiCBQvIyso6KuiHDBlCcXEx48aN45Of/CQ33HADt956a4fjPUzX0csRD7+ykUF9Uvj61edywcDM7u7OKTt8eeZhd0wezMTBWdw25x0WbKzg7S37uHJUX75+1bkU9klhz4F6kuJD7Kg8xN827+WWCQWkJep6felcEyZMOOo690cffZSnn34agB07drBp0yaysrKO2mfQoEGMGzcOgAsuuICSkpJO64+C/iyyeOs+1u+u4YHrRzJ1dG53d6fLnJffi3f/7Ur2HWzkO8+t5dkVZWwpr2XKuTnMXrjlqDtyn1+1mz/MnKiF1gLkREfep0tKSsqR92+88QZ//etfefvtt0lOTmbKlCnt3sGbkJBw5H0oFKKurq7T+qOrbs4S1XVN/PMT7zIwK5np4z64dk3QxMQY2WkJPHbLeB6ZMY4dlYeYtWAL52Sn8pFzsrj3Y8N44PqRrNhRxbefWU1PnMKUM0daWho1NTXt1lVXV5ORkUFycjLr169n8eLFp7l3URzRm9kA4DdAP6AVmO3ujxzTxoBHgGuAQ8Dn3P3dSN3USF0I+IW7/1enjkBOqLG5ldkLt/DzhVupqW/m1/80gYyU+O7u1mk1fVwe156XS019M72T4466fHRvbSM/fn0zzywv49aJA5k8LJsJhZkkxesIX6KXlZXFpEmTGD16NElJSfTt2/dI3dSpU5k1axZjxozh3HPPZeLEiae9fx2ejDWzXCDX3d81szRgGXCju69t0+Ya4MuEg/4i4BF3v8jMQsBG4EqgFFgK3NJ23/boZGzncHcefH4dj7+1DYCJgzN5cubF3dyrnqWl1fnRSxtYtGUvK0urAeiXnshVo/qSn5HETePzOdjQTG7vRBJiFf49VXsnJ4Os00/GuvsuYFfkfY2ZrQPygLZhPR34jYf/1lhsZr0jf0EUApvdfWukI09G2p4w6KVz/G7xdh5/axuTh2Vz0/j+TByc1fFOZ5lQjHHftOEALNlWyc8XbOHV9eX85u3w/QXfm78egMHZKVwzOpete2sB+OrHhulqHjljnNTJWDMrBMYD7xxTlQfsaLNdGilrr/yi43z2TGAmQEFBwcl0S47R2NzKI69u5Cevb2FkbjpzPlukx/tFYcKgTC4szODB59eR1zuJpPgQ63YdoCAzmUde3cSPX99MQWYyu6rreH19BfdfP5J/OD9fSytLjxd10JtZKvAX4KvufuxFqu3dM+8nKP9goftsYDaEp26i7Zccra6xhc/MeYelJfu5cmRfvn3dSIX8STAz/v26kR8o/9RFBdQ3tZKZEk/5gXru/v173PfUKh55dRMP3TyWSZEnbYn0RFEFvZnFEQ75J9z9qXaalAID2mznA2VA/HHKpYs8+Pxalpbs5+FPjuWm8Xla9KuTJMfHkhw5h52TnsjcmRNZsLGc781fz62Pv8PnPlLI5cNz+MWb29i0p4ZReb2489LBLNm2n4bmFvr3SmJ8QW82l9fSJy2BEbnppGqZBjlNornqxoDHgXXu/vBxms0D7o7MwV8EVLv7LjOrAIaa2SBgJzAD+FTndF0Ann6vlKE5aQzvl8Yfi0t54p33+eLkwfzD+VrWtyuFYozLh/dl4uAsvvPsWn69qIRf/q0EgMKsZF5Zu4dX1u457v45aQn823Ujue68XD03V7pcNIcUk4DbgFVmtjxS9q9AAYC7zwLmE77iZjPhyytvj9Q1m9ndwEuEL6+c4+5rOnUEZ5G3t+xjTVk1Q3JSuXRYNs+u3MW9f1hxVJtLh2XzL1ed2009PPskx8fyXx8fwzemDufZlWUMzUnj4nOyeH/fId59fz/nF2SQlRrPxj01LC2pxB1izHhmxU7umfseX33yPW6ZUMDkYdks2VZJWmIs775fRVZKPFPOzeay4Tmk685dOUVa66ab7awK3/3W0NTC5vLayC97HBMGZXKgvomPDu1DWVU9j7+1lblL/n5ee2x+L9buOsCQnDTW7QqfMvnKFUP58uVDNCd/BmhpdZ5bWcYLq3bz4prdx21XkJnMn+68mL7piaexd2eeM/HyytTUVGpraykrK+Oee+7hz3/+8wfaTJkyhYceeoiioqOvmtRaN2eQNWXVR56bGo0hOancND6PVaXVvLJuDxnJ8fzq9gtZVVpNiztXt1nzRXq2UIwxfVwe08flUVxSya7qekb2TyczOZ49NfVkpybw5qa93PfUSib/8HWKCjO49aKBXFCYQXpinJZsCJD+/fu3G/KdqUcG/aqd1fz4tU3cffnQ7u5Kl5m3ooxFm/cCMG10P64aFb6T7mMj+rJt70He3LSXA/VNLCvZT3xsDFeP6sfNRfkkx4f/l+2rbcDMyEyJp+9IHe2dyYoKj15c7vCdyzeOz2N0Xi9++bdtvLRmD3c98S4AyfEhRuSmc2FhJteNycUMhvVNI07/kutW3/jGNxg4cOCRNekfeOABzIyFCxeyf/9+mpqaePDBB5k+ffpR+7Vd+bKuro7bb7+dtWvXMmLEiE5b76ZHTt0k5A713M/+NyX/dW13d6VTLdu+nz0H6umbnsjHf7YICK+dvuibV3Rzz6SnO9TYzJub9vLu9v2U1zSwpaL2yJ28APGxMXz24oHsrKpjVP9ejBvQmxG56WSmxNPQ3BL4u3qPmsp44T7Yvapzv6DfeTDtxKu3vPfee3z1q19lwYIFAIwcOZIXX3yR3r17k56ezt69e5k4cSKbNm3CzI5M3bQN+ocffpjVq1czZ84cVq5cyfnnn8/ixYvPjqmbxuZWHD9j/7CuKatm7pL3eeKd9zn892paQiwTBmUyqn9693ZOzgjJ8bFcParfUdNzG/fUsGJHFZUHG/ndO9v5nzfDS13MX/X3Of8+qfHsrW1kWN9UrhjRl9gY4+Pn59MnLUGXd3ay8ePHU15eTllZGRUVFWRkZJCbm8u9997LwoULiYmJYefOnezZs4d+/dqfZl24cCH33HMPAGPGjGHMmDGd0rce/X+6pdWJMbjq/y1gb20jz9w96cgThjpTa6uz/1AjvZLieH7VLpbvqOKqkf24+JxTWzLgpTW7mb1wK8u27ycpLsRVI/tiGC+u2c3I/uk8/rkLO2kEcjYa1jftyDIMNxcN4LmVZUwZlkNSfIgNu2t46t1SahqayU5LoLikkp+9sQWAx17bDMCo/uncPmkQN47rz8HGFh5/cytXj+7HyNz0M/v+iw6OvLvSJz7xCf785z+ze/duZsyYwRNPPEFFRQXLli0jLi6OwsLCdpcobqsr/tv36KD/1aISrhzRl5J9hwD4/vx1/OKznROO7s78VbtZt+sA81fvYmvFwSN1ZvDLv5UwODuFEbnp3HP5UM7tF/26Ji2tzkMvbzjyizW8Xxpz75hIRko8S7ZV8uKa3Qw/ic8T6UhmSjyfubjwyHZ2WgKXDD36bt2dVXVU1jby2vpymltb+eu6cv7Pn1bw2GubaG5xdlbV8ehrm8ntlcitEwfyqQkFZ91Kp6dqxowZ3HHHHezdu5cFCxbwxz/+kZycHOLi4nj99dfZvn37CfefPHkyTzzxBJdddhmrV69m5cqVndKvHh30331uLd99Lrz+2fVj+/PsijLe2FDOlHNzTulz3Z3fL3mfbz29Ggif3Pr61eeyaU8NvZPj+cbU4fxh6fv85u3tPL9yF0u3VXLrxIG8um4PYwf05o6PDiYjJZ79BxtJT4ojNSGWUIzR2NxKKMb42+a9/OyNLVwwMIPHP1tEYlzoyFUSFxZm8PPbLuCjQ3XLvJxeeb2TyOudxHn5vQD42pXDeHntHn76xhZ2V9fx0M1jqWtq4YnF2/nRSxt4+JWNjO6fTlZqAknxIdyd3F5JTB3dj+H90vRkrnaMGjWKmpoa8vLyyM3N5dOf/jTXX389RUVFjBs3juHDh59w/7vuuovbb7+dMWPGMG7cOCZM6JxHffbYk7Hf//VzPBr5J2ZCbAzv/vuV3PDjt6hrbOHFeyd/qJtIDtQ38f6+Q3xi1iLqm1q5YGAGj8wYhzvtPki6qaWV1Tur+Zc/rWBrxUEykuPYf6jpA+2S4kLcMLY/z6/ahbtzsLGF9MRYlnzrY7oMTs5I63Yd4LmVZSwt2c/u6noONjQTijGq6ppobG6ld3Ic/zptBJ+8cEDHH3YanInX0Z+Kkz0Z2yODPiVvmB/cuZE3N1Wwt7aBq0f1Izk+lvfe38/Hf7aInLRE7ppyDpOHZTMwM5mK2obj3lBSXlPP2rIDPL9yF395t/Sox8jNu3sSY/J7d9if5pZWSvYdYnCfFN7ZVsljr21i0ZZ95PVO4pYJA/jrunJW7axmdF4vzumTwitr9/DADaP4+AVahkCCpfpQE29sDC/jvGz7fiYMyuSfJg0Kn38y2FFZx7Z9BxncJ4VeyXGn7a5eBf0ZGPSDR4zxrevan5t6Y0M5M3+7jMbmVgD6pidQXtPAb//pog/MST6zfCdf//NKGptbiY0xrjkvl15JcUwf15+MlPhTOrG7r7aB2FAMvZLCf5Dd/chJlLbvRYLo8HmoecvL2FlVx8CsZGJjjC3HnOsampNKXCiGK0b05abxeawpq2ZvTQM3jMsjsxPn/xX0Z2DQd7QEwuqd1WypqOU7z64lPzOZFTuqyO2VyHNfvoSHXt5A0cBMRuWlc92jbzG+oDf/eGEBV47oS69kzSmKdKbmllbmr97NE4u3kxAX4orhOeT2SmRXdT37DzWybPt+GptbeWdb5Qf2zU5L4DMTB/LxC/LJ7ZVIS6vT2NJ65KbAk7Fu3TqGDx9+VhxguTvr168PftAfq7ikkk/MepuCzGTerzx0pDwrJZ6/fu1SXTkg0s22VNSydFsl/XsnkZEcz8JNFSzZVsmCjRVAeDXPgw3NNLa0cnPRAEb0S+PyEX3J650U1edv27aNtLQ0srKyAh327s6+ffuoqalh0KBBR9UFPujdnYu+9yrlNQ3EhYz0xDiG5KTy9avP/cDt5SLSc6zeWc3SkkqWbd9PZko8tQ3NPLO8jJZWJxRjDO6TQlFhJl+acg79eycROs6Szk1NTZSWlnZ4jXoQJCYmkp+fT1zc0TMUgQ96gPufWc2v397Oz2+7QIt7iZzBDtQ3sauqnudXlrF21wHe2ryX+qbwOblR/dOZOqofk4dlM3ZAxxdSnE3OiqBvbmmlvrlVt3WLBMzOqjq+9/w6VpRWkRwfYuOe8APaC7OSiTEjPzOZSedk0dzqvLRmN0NyUhk3oDepCbEkxYXo3zuJPmkJ5KQlEDLjQH0T6YlxgXvgy1kR9CJydqg+1MQTS7azYkcV9U2trCytOnJ/S7/0RBqaW9q936VfeiKxIaN0fx3J8SE+OrQPFxZmkp+RjBkMyAjfS5OTnkCf1ITTOqbOcEqLmpnZHOA6oNzdR7dT/3Xg020+bwSQ7e6VZlYC1AAtQPPxOiEiEq1eyXF8acqQI9utrU5NQzMNTS1kpyXgDhW1DdQ1tnCosYVN5TXsq21kwcYK4mNj+NRFBWzfe4gFGyt4ac0HH/cYY3BhYSaThvQhPyOJCwZmUBC5X2dHZR0jcsNLQp9Jy0J3eERvZpOBWuA37QX9MW2vB+5198sj2yVAkbvvPZlO6YheRLpaS6uz72A4vCE8RXSgronymgZeXL3ryBQRhNcSqjzYeGQ7PjaGQVkpDOuXxvVjcvno0GzMoOpQE7uq66htaCYjOZ70xDjMID8jqcuvBjqlI3p3X2hmhVF+1y3A3Oi7JiLSPUIxRk5aIjlp4bvqLxiYcaTua1cOo76phZJ9Byku2c/SkkpizLhiRA6by2s5UNfM1r21vLWpgmdXlBFjHHXX/bGG90ujb3oiG3bX0NzqnJeXzuRh2WSnJXDBwAyamp3S/YdobGlldF4vkuNDxIViaGn1TllGJao5+kjQP3eiI3ozSwZKgSHuXhkp2wbsBxz4ubvPPsH+M4GZAAUFBRd0tMqbiEh3a2ppPXI/QGpCLGmJsaQnxpGWGEurw4G6Jg42NvPUuztpbG5lVP906ppa2LC7hq17D3b4+bExxvkFGRRkJdO/dxKby2vok5rAuAG9WbGjipSEWGJDMRRkJnNz0YBTOxkbZdD/I3Cru1/fpqy/u5eZWQ7wCvBld1/Y0fdp6kZEgq50/yG2Vhxk1c5qslMTyM9IwoH1u2toaG7hYEMzlQeb2FJey/uVh9h9oJ6M5Diq65podYgLGU0tjhm4w/YfXHdanjA1g2Ombdy9LPKz3MyeBiYAHQa9iEjQ5Wckk5+RzORh2UeVTxrS/hLmDc0txIdiOFDfzNqyA4zsn058KAYz2FVdz+AfHP+7OuW0sZn1Ai4FnmlTlmJmaYffA1cBqzvj+0REzjYJsSHMjF5JcVx8Tha9kuJIig8/62JQn5QT7hvN5ZVzgSlAHzMrBe4H4gDcfVak2U3Ay+7edtKpL/B05ExzLPB7d3/xJMcmIiKnKJqrbm6Jos2vgF8dU7YVGPthOyYiIp3jzLniX0REPhQFvYhIwCnoRUQCTkEvIhJwCnoRkYBT0IuIBJyCXkQk4BT0IiIBp6AXEQk4Bb2ISMAp6EVEAk5BLyIScAp6EZGAU9CLiAScgl5EJOA6DHozm2Nm5WbW7tOhzGyKmVWb2fLI69tt6qaa2QYz22xm93Vmx0VEJDrRHNH/CpjaQZs33X1c5PUdADMLAT8BpgEjgVvMbOSpdFZERE5eh0Hv7guByg/x2ROAze6+1d0bgSeB6R/ic0RE5BR01hz9xWa2wsxeMLNRkbI8YEebNqWRMhEROY06fGZsFN4FBrp7rZldA/wvMBSwdtr68T7EzGYCMwEKCgo6oVsiIgKdcETv7gfcvTbyfj4QZ2Z9CB/BD2jTNB8oO8HnzHb3Incvys7OPtVuiYhIxCkHvZn1MzOLvJ8Q+cx9wFJgqJkNMrN4YAYw71S/T0RETk6HUzdmNheYAvQxs1LgfiAOwN1nAZ8A7jKzZqAOmOHuDjSb2d3AS0AImOPua7pkFCIiclwWzuSepaioyIuLi7u7GyIiZwwzW+buRe3V6c5YEZGAU9CLiAScgl5EJOAU9CIiAaegFxEJOAW9iEjAKehFRAJOQS8iEnAKehGRgFPQi4gEnIJeRCTgFPQiIgGnoBcRCTgFvYhIwCnoRUQCTkEvIhJwHQa9mc0xs3IzW32c+k+b2crIa5GZjW1TV2Jmq8xsuZnpSSIiIt0gmiP6XwFTT1C/DbjU3ccA3wVmH1N/mbuPO96TT0REpGt1+MxYd19oZoUnqF/UZnMxkH/q3RIRkc7S2XP0nwdeaLPtwMtmtszMZp5oRzObaWbFZlZcUVHRyd0SETl7dXhEHy0zu4xw0F/SpniSu5eZWQ7wipmtd/eF7e3v7rOJTPsUFRX1vCeWi4icoTrliN7MxgC/AKa7+77D5e5eFvlZDjwNTOiM7xMRkeidctCbWQHwFHCbu29sU55iZmmH3wNXAe1euSMiIl2nw6kbM5sLTAH6mFkpcD8QB+Dus4BvA1nAT80MoDlyhU1f4OlIWSzwe3d/sQvGICIiJxDNVTe3dFD/BeAL7ZRvBcZ+cA8RETmddGesiEjAKehFRAJOQS8iEnAKehGRgFPQi4gEnIJeRCTgFPQiIgGnoBcRCTgFvYhIwCnoRUQCTkEvIhJwCnoRkYBT0IuIBJyCXkQk4BT0IiIBp6AXEQm4DoPezOaYWbmZtfsYQAt71Mw2m9lKMzu/Td1UM9sQqbuvMzsuIiLRieaI/lfA1BPUTwOGRl4zgZ8BmFkI+EmkfiRwi5mNPJXOiojIyesw6N19IVB5gibTgd942GKgt5nlAhOAze6+1d0bgScjbUVE5DTqjDn6PGBHm+3SSNnxyttlZjPNrNjMiisqKjqhWyIiAp0T9NZOmZ+gvF3uPtvdi9y9KDs7uxO6JSIiALGd8BmlwIA22/lAGRB/nHIRETmNOuOIfh7wmcjVNxOBanffBSwFhprZIDOLB2ZE2oqIyGnU4RG9mc0FpgB9zKwUuB+IA3D3WcB84BpgM3AIuD1S12xmdwMvASFgjruv6YIxiIjICXQY9O5+Swf1DvzzcermE/6LQEREuonujBURCTgFvYhIwCnoRUQCTkEvIhJwCnoRkYBT0IuIBJyCXkQk4BT0IiIBp6AXEQk4Bb2ISMAp6EVEAk5BLyIScAp6EZGAU9CLiAScgl5EJOAU9CIiARdV0JvZVDPbYGabzey+duq/bmbLI6/VZtZiZpmRuhIzWxWpK+7sAYiIyIlF8yjBEPAT4ErCDwJfambz3H3t4Tbu/iPgR5H21wP3untlm4+5zN33dmrPRUQkKtEc0U8ANrv7VndvBJ4Epp+g/S3A3M7onIiInLpogj4P2NFmuzRS9gFmlgxMBf7SptiBl81smZnNPN6XmNlMMys2s+KKiooouiUiItGIJuitnTI/Ttvrgb8dM20zyd3PB6YB/2xmk9vb0d1nu3uRuxdlZ2dH0S0REYlGNEFfCgxos50PlB2n7QyOmbZx97LIz3LgacJTQSIicppEE/RLgaFmNsjM4gmH+bxjG5lZL+BS4Jk2ZSlmlnb4PXAVsLozOi4iItHp8Kobd282s7uBl4AQMMfd15jZnZH6WZGmNwEvu/vBNrv3BZ42s8Pf9Xt3f7EzByAiIidm7sebbu8+RUVFXlysS+5FRKJlZsvcvai9Ot0ZKyIScAp6EZGAU9CLiAScgl5EJOAU9CIiAaegFxEJOAW9iEjAKehFRAJOQS8iEnAKehGRgFPQi4gEnIJeRCTgFPQiIgGnoBcRCTgFvYhIwEUV9GY21cw2mNlmM7uvnfopZlZtZssjr29Hu6+IiHStDp8wZWYh4CfAlYSfH7vUzOa5+9pjmr7p7td9yH1FRKSLRHNEPwHY7O5b3b0ReBKYHuXnn8q+IiLSCaIJ+jxgR5vt0kjZsS42sxVm9oKZjTrJfTGzmWZWbGbFFRUVUXRLRESiEU3QWztlxz5o9l1goLuPBR4D/vck9g0Xus929yJ3L8rOzo6iWyIiEo1ogr4UGNBmOx8oa9vA3Q+4e23k/Xwgzsz6RKJ9CpoAAAbdSURBVLOviIh0rWiCfikw1MwGmVk8MAOY17aBmfUzM4u8nxD53H3R7CsiIl2rw6tu3L3ZzO4GXgJCwBx3X2Nmd0bqZwGfAO4ys2agDpjh7g60u28XjUVERNph4TzuWYqKiry4uLi7uyEicsYws2XuXtRene6MFREJOAW9iEjAKehFRAJOQS8iEnAKehGRgFPQi4gEnIJeRCTgFPQiIgGnoBcRCTgFvYhIwCnoRUQCTkEvIhJwCnoRkYBT0IuIBJyCXkQk4BT0IiIBF1XQm9lUM9tgZpvN7L526j9tZisjr0VmNrZNXYmZrTKz5Wamp4mIiJxmHT5K0MxCwE+AKwk/7Hupmc1z97Vtmm0DLnX3/WY2DZgNXNSm/jJ339uJ/RYRkShFc0Q/Adjs7lvdvRF4EpjetoG7L3L3/ZHNxUB+53ZTREQ+rGiCPg/Y0Wa7NFJ2PJ8HXmiz7cDLZrbMzGYebyczm2lmxWZWXFFREUW3REQkGh1O3QDWTlm7TxQ3s8sIB/0lbYonuXuZmeUAr5jZendf+IEPdJ9NeMqHoqKinvfEchGRM1Q0R/SlwIA22/lA2bGNzGwM8AtgurvvO1zu7mWRn+XA04SngkRE5DSJJuiXAkPNbJCZxQMzgHltG5hZAfAUcJu7b2xTnmJmaYffA1cBqzur8yIi0rEOp27cvdnM7gZeAkLAHHdfY2Z3RupnAd8GsoCfmhlAs7sXAX2BpyNlscDv3f3FLhmJiIi0y9x73nR4UVGRFxfrknsRkWiZ2bLIAfYH6M5YEZGAU9CLiAScgl5EJOAU9CIiAaegFxEJOAW9iEjAKehFRAJOQS8iEnAKehGRgFPQi4gEnIJeRCTgFPQiIgGnoBcRCTgFvYhIwCnoRUQCTkEvIhJwUQW9mU01sw1mttnM7mun3szs0Uj9SjM7P9p9RUSka3UY9GYWAn4CTANGAreY2chjmk0DhkZeM4GfncS+IiLShaI5op8AbHb3re7eCDwJTD+mzXTgNx62GOhtZrlR7isiIl2ow4eDA3nAjjbbpcBFUbTJi3JfAMxsJuF/DQA0mNnqKPp2pusD7O3uTpwGGmewnC3jhDNrrAOPVxFN0Fs7Zcc+Ufx4baLZN1zoPhuYDWBmxcd7yG2QaJzBonEGT1DGGk3QlwID2mznA2VRtomPYl8REelC0czRLwWGmtkgM4sHZgDzjmkzD/hM5OqbiUC1u++Kcl8REelCHR7Ru3uzmd0NvASEgDnuvsbM7ozUzwLmA9cAm4FDwO0n2jeKfs3+MIM5A2mcwaJxBk8gxmru7U6Zi4hIQOjOWBGRgFPQi4gEXI8K+iAtl2Bmc8ysvO39AGaWaWavmNmmyM+MNnXfjIx7g5ld3T29PnlmNsDMXjezdWa2xsy+EikP1FjNLNHMlpjZisg4/yNSHqhxHmZmITN7z8yei2wHdZwlZrbKzJabWXGkLHhjdfce8SJ8snYLMJjwZZkrgJHd3a9TGM9k4HxgdZuyHwL3Rd7fB/wg8n5kZLwJwKDIf4dQd48hynHmAudH3qcBGyPjCdRYCd8Tkhp5Hwe8A0wM2jjbjPdrwO+B5yLbQR1nCdDnmLLAjbUnHdEHarkEd18IVB5TPB34deT9r4Eb25Q/6e4N7r6N8NVLE05LR0+Ru+9y93cj72uAdYTviA7UWD2sNrIZF3k5ARsngJnlA9cCv2hTHLhxnkDgxtqTgv54yygESV8P319A5GdOpDwQYzezQmA84aPdwI01Mp2xHCgHXnH3QI4T+G/g/wKtbcqCOE4I/2X9spktiyzDAgEcazR3xp4uUS+XEEBn/NjNLBX4C/BVdz9g1t6Qwk3bKTsjxuruLcA4M+sNPG1mo0/Q/Iwcp5ldB5S7+zIzmxLNLu2U9fhxtjHJ3cvMLAd4xczWn6DtGTvWnnREH81SC2e6PZFVPYn8LI+Un9FjN7M4wiH/hLs/FSkO5FgB3L0KeAOYSvDGOQm4wcxKCE+fXm5mvyN44wTA3csiP8uBpwlPxQRurD0p6M+G5RLmAZ+NvP8s8Eyb8hlmlmBmgwiv67+kG/p30ix86P44sM7dH25TFaixmll25EgeM0sCPgasJ2DjdPdvunu+uxcS/h18zd1vJWDjBDCzFDNLO/weuApYTQDH2u1ng485230N4as2tgDf6u7+nOJY5gK7gCbCRwKfB7KAV4FNkZ+Zbdp/KzLuDcC07u7/SYzzEsL/fF0JLI+8rgnaWIExwHuRca4Gvh0pD9Q4jxnzFP5+1U3gxkn4Cr8Vkdeaw5kTxLFqCQQRkYDrSVM3IiLSBRT0IiIBp6AXEQk4Bb2ISMAp6EVEAk5BLyIScAp6EZGA+/+fEWqHip+AZgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.fit_one_cycle(1,3.02E-03,moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner.save('first_cycle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "seed_all(seed)\n",
    "learner.load('first_cycle');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner.freeze_to(-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='' max='1' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.00% [0/1 00:00<00:00]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>error_rate</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='69' class='' max='575' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      12.00% [69/575 00:03<00:24 5.6063]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n",
      "Min numerical gradient: 5.75E-06\n",
      "Min loss divided by 10: 1.58E-07\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deZhcdZ3v8fe3ek96y9JZOkknJCSEJJAEwiaKMAiyg6ODyuLosFxGZwDHURmv28zozFzviIKgCKMyyKJXFgVFR0AgICQh+0r2dKfTTXpLeqve63v/qOrQJL0l6VNV3fV5PU896Trnd6q+5zyV+tQ553d+x9wdERFJXaFEFyAiIomlIBARSXEKAhGRFKcgEBFJcQoCEZEUl57oAo7W+PHjfcaMGYkuQ0RkWFm1alWNuxf1Nm/YBcGMGTNYuXJlossQERlWzKy0r3k6NCQikuIUBCIiKU5BICKS4hQEIiIpTkEgIpLiFAQiIilOQSAikuIUBCIiw8A9L27n9e01gby2gkBEJMm1d0a456VtrNhTF8jrKwhERJLcvoMtRBymjx0VyOsrCEREklxpbTMAJeMUBCIiKWlvXRjQHoGISMoqrQ2TnRGiKC8rkNdXEIiIJLmyujAlY0dhZoG8voJARCTJdQdBUBQEIiJJzN1jQTA6sPdQEIiIJLGapnbC7V2UjM0J7D0CCwIzm2ZmL5vZFjPbZGZ39NLmejNbH3u8YWYLg6pHRGQ4KuvuMTQuuD2CIG9V2Ql8wd1Xm1kesMrMXnD3zT3a7AY+6O4HzOxS4EHgrABrEhEZVsrqotcQTAvwHEFgQeDulUBl7O9GM9sCTAE292jzRo9FlgFTg6pHRGQ4KqttwQymjhmGh4Z6MrMZwGJgeT/NbgJ+38fyt5rZSjNbWV1dPfQFiogkqdK6ZiblZ5OdkRbYewQeBGaWCzwF3OnuDX20uYBoEHy5t/nu/qC7L3H3JUVFRcEVKyKSZPYG3HUUAg4CM8sgGgKPufvTfbQ5Ffgv4Gp3rw2yHhGR4aa0Nsz0gMYY6hZkryEDfgJscfe7+2hTAjwN3Oju24KqRURkOGpp76KqsS3wPYIgew2dC9wIbDCztbFpXwFKANz9AeDrwDjgh7FLpzvdfUmANYmIDBt7D0S7jpYE2HUUgu019DrQ78AY7n4zcHNQNYiIDGdltbEgGM7nCERE5NiVBjz8dDcFgYhIkiqrbSYvK53CURmBvo+CQEQkSZXVhSkZF9zw090UBCIiSao0DtcQgIJARCQpRSJOeV1LYPcp7klBICKShN5paKW9K6I9AhGRVHVo+OkAb0jTTUEgIpKE4nUNASgIRESSUlldmLSQUVyYHfh7KQhERJJQaV2YKYU5pKcF/zWtIBARSUJldcGPOtpNQSAikoTKapsDvT1lTwoCEZEk09DawYFwR+BjDHVTEIiIJJl49hgCBYGISNLZW9d9HwIFgYhISuoeflp7BCIiKaqsLszY0ZnkZQc7/HQ3BYGISJIpqw3HrccQKAhERJJOWV04bj2GQEEgIpJUOroi7DvYErfzA6AgEBFJKpUHW+mKeNx6DIGCQEQkqZTWNQPx6zEECgIRkaRy6D4E2iMQEUlNZbVhMtNDTMwLfvjpbgoCEZEkUlYXZtqYHEIhi9t7KghERJJIaW2Y6eOCvz1lTwoCEZEk4e7srQvH9UQxKAhERJLGgXAHjW2dCgIRkVT1+o4aAOZMzIvr+yoIRESSgLvz4NKdzCwazftmjYvrewcWBGY2zcxeNrMtZrbJzO7opc1cM3vTzNrM7B+DqkVEJNm9ubOWjfsauPUDM+PaYwggPcDX7gS+4O6rzSwPWGVmL7j75h5t6oDbgWsCrENEJOk9sHQXRXlZXLN4StzfO7A9AnevdPfVsb8bgS3AlMPaVLn7W0BHUHWIiCS7zRUNLN1WzaffN4PsjLS4v39czhGY2QxgMbD8GJe/1cxWmtnK6urqoSxNRCThHnptF6Mz07jhrOkJef/Ag8DMcoGngDvdveFYXsPdH3T3Je6+pKioaGgLFBFJoPIDYZ5dV8EnziyhYFR87kh2uECDwMwyiIbAY+7+dJDvJSIyHP309T0Y8DfvPyFhNQTZa8iAnwBb3P3uoN5HRGS4qg938Iu3yrhyYTFTCnMSVkeQvYbOBW4ENpjZ2ti0rwAlAO7+gJlNAlYC+UDEzO4E5h3rISQRkeHk0eWlhNu7uPW8mQmtI7AgcPfXgX47w7r7O8DUoGoQEUlWrR1d/OzPezhvThEnT85PaC26slhEJAGeWbOPmqY2bkvw3gAoCERE4i4ScR5auotTphRwTpyHk+iNgkBEJM5e3VbNrppmbj1vJtF+NYmlIBARibP15fWYwUXzJia6FEBBICISd6V1zUzKz07IcBK9URCIiMRZWW3870LWHwWBiEicldaFmT5OQSAikpKa2zqpbmyL+w3q+6MgEBGJo7K6MID2CEREUlVpbSwIxmqPQEQkJZXVNQNQoj0CEZHUtKc2TOGoDApyEnPvgd4oCERE4qisNpxUJ4pBQSAiEleldc1MT6JrCEBBICISNx1dESoOtiZVjyFQEIiIxM2+Ay10RTyprioGBYGISNzsqY32GNI5AhGRFNV9MdkMHRoSEUlNpbVhcjLSKMrLSnQp76EgEBGJk9LYqKPJcDOanhQEIiJxUlrbnFRXFHdTEIiIxEEk4pTVhZPuGgJQEIiIxEVVYxttnRGmj0+uHkOgIBARiYvS7q6j2iMQEUlNpUl4H4JuCgIRkTgorW0mLWQUF+YkupQjKAhEROKgtDbMlMIcMtKS72s3+SoSERmBypLshvU9KQhEROKgtFZBICKSsg6G26lv6Uiq+xT3FFgQmNk0M3vZzLaY2SYzu6OXNmZm95rZDjNbb2anBVWPiEiidN+wPhmvKgZID/C1O4EvuPtqM8sDVpnZC+6+uUebS4HZscdZwI9i/4qIjBjJ3HUUAtwjcPdKd18d+7sR2AJMOazZ1cAjHrUMKDSzyUHVJCKSCGWxi8mS7YY03eJyjsDMZgCLgeWHzZoC7O3xvJwjw0JEZFgrrQ0zIS+LUZlBHoQ5doMKAjMbbWah2N9zzOwqM8sY5LK5wFPAne7ecPjsXhbxXl7jVjNbaWYrq6urB/O2IiJJI5l7DMHg9wiWAtlmNgV4CfgM8PBAC8XC4ingMXd/upcm5cC0Hs+nAhWHN3L3B919ibsvKSoqGmTJIiLJobSumZIk7TEEgw8Cc/cw8JfAD9z9I8C8fheI3nnhJ8AWd7+7j2bPAp+K9R46G6h398pB1iQikvRaO7rY39CW1HsEgz1gZWZ2DnA9cNMglz0XuBHYYGZrY9O+ApQAuPsDwPPAZcAOIEx0T0NEZMQoS/IeQzD4ILgT+CfgGXffZGYzgZf7W8DdX6f3cwA92zjwuUHWICIy7OypiQ0/PS55Dw0NKgjc/VXgVYDYSeMad789yMJEREaCQ3sESdp1FAbfa+hxM8s3s9HAZmCrmX0x2NJERIa/0towednpFI4aVEfLhBjsyeJ5sa6f1xA9rl9C9Pi/iIj0ozQ26mi0/0xyGmwQZMS6gl4D/MbdO+ilv7+IiLxXWW1zUp8fgMEHwY+BPcBoYKmZTQcOvzhMRER66OyKUH6gJanPD8DgTxbfC9zbY1KpmV0QTEkiIiNDxcFWOiOe1F1HYfAniwvM7O7uYR7M7LtE9w5ERKQPpXXdg80l99flYA8N/RRoBK6NPRqAnwVVlIjISNB9H4Jk3yMY7AVls9z9oz2e/3OPq4VFRKQXu6qbyUwPMSk/O9Gl9GuwewQtZvb+7idmdi7QEkxJIiLD38FwO0+tLuecmeMIhZK36ygMfo/gNuARMyuIPT8A/HUwJYmIDH/3vrSDxtYO7rp0bqJLGdBgew2tAxaaWX7seYOZ3QmsD7I4EZHhaFd1E4+8uYePnzGNkyfnJ7qcAR3VHcrcvaHHzWX+IYB6RESGvX///dtkpYf4/EVzEl3KoBzPrSqT+6CXiEgCvLGzhhc27+ezF5zIhLzkPknc7XiCQENMiIj00BVxvvXbLUwpzOGm95+Q6HIGrd9zBGbWSO9f+AbkBFKRiEiS2lRRz+SCHMaOzux1/lOry9lc2cA9n1hEdkZanKs7dv0GgbvnxasQEZFkVtfcztX3/ZlRmWl8/qI53HD2dDLS3j2o0tzWyX/+z1YWlxRy1cLiBFZ69I7n0JCISMpYsbuWzogzZcwo/vm5zVx2z2u8vr3m0Pwfv7qTqsY2vnr5vKQecro3CgIRkUFYtquOnIw0fvO5c3nwxtNp64xww0+Wc8sjK1m2q5YHX9vFlQuLOX36mESXetQGe0GZiEhKW7arltOnjyEzPcTF8ydx3pwifvL6bu5/eQcvbN5PZnqIL19yUqLLPCYKAhGRARwMt7N1fyOXnzL50LTsjDQ+d8GJfPS0qfzgT9uZX1zA1DHJPbhcXxQEIiIDWL67Dnc4e9a4I+ZNKsjm2x85JQFVDR2dIxARGcDyXXVkpYc4dWrBwI2HIQWBiMgAlu+u5bSSMWSlD59rA46GgkBEpB/14Q42VzZw1syxiS4lMAoCEZF+vLUndn5g5pHnB0YKBYGISD+W764lMz3EommFiS4lMAoCEZF+LN9dx6JphcNq7KCjpSAQEelDQ2sHG/fVj+jDQqAgEBHp06o9B4g4nH3CyD1RDAoCEZE+LdtdS0aasbhk+I0fdDQCCwIz+6mZVZnZxj7mjzGzZ8xsvZmtMLMFQdUiInIslu2qY+HUQnIyR+75AQh2j+Bh4JJ+5n8FWOvupwKfAu4JsBYRkaPS1NaZEucHIMAgcPelQF0/TeYBL8Xavg3MMLOJQdUjInI0VpUeoCviI/pCsm6JPEewDvhLADM7E5gOTO2toZndamYrzWxldXV1HEsUkVS1bFct6SEblvcXOFqJDIL/AMaY2Vrg74E1QGdvDd39QXdf4u5LioqK4lmjiKSo5btqOXVqAaMyR/4gzQlbQ3dvAD4DYNH7uu2OPUREEirc3sn68npuOW9mokuJi4TtEZhZoZllxp7eDCyNhYOISEKtLj1IZ8Q5a4RfP9AtsD0CM3sCOB8Yb2blwDeADAB3fwA4GXjEzLqAzcBNQdUiInI0lu2qJS1kLJmhIDgu7v7JAea/CcwO6v1FRI7V8t21LJhSQG7WyD8/ALqyWETkPVrau1i3t37EDyvRk4JARKSHJ1aU0d4V4Zxe7k88UikIRERi3tpTx789v4UPnTyB82anTld1BYGICFDV0MpnH1vN1DE5fPfaRYRCluiS4iY1zoSIiPSjvTPCZx9bTVNrJ4/edBYFORmJLimuFAQikvL+7fktrCw9wL2fXMxJk/ISXU7c6dCQiKS0Z9aU8/Abe7jp/Sdw1cLiRJeTEAoCEUlZmysa+KenN3DmCWO569K5iS4nYRQEIpKS6sMd3PboKgpyMrj/utPISEvdr8PUXXMRSVmV9S18/ME3qaxv4YfXn05RXlaiS0oonSwWkZSyqaKev3n4LZrbuvjpp89IifsNDERBICIp45WtVXzusdXk52Tw5N+ew9xJ+YkuKSkoCEQkJTy+vIyv/WYjJ03M42efOYOJ+dmJLilpKAhEZESLRJz/+8et/OiVnZx/UhH3XXdayowqOljaGiIyYrk7//vXG3liRRnXnVXCv1w1n/QU7h3UFwWBiIxYP/vzHp5YUcbfnj+LL334JKJ3xZXDKRpFZERauq2ab/1uMx+eP5EvXqwQ6I+CQERGnF3VTfzd46uZMzGPu1NsJNFjoSAQkRGlobWDmx9ZSXpaiIc+tYTROjE8IAWBiIwYXRHn9ifWUFYb5ofXn8a0saMSXdKwoKgUkRHjO394m1e2VvPtjyzg7Jmpc6vJ46UgEJFhryviPL6ijB8v3cWNZ0/n+rOmJ7qkYUVBICLD1o6qRp5ctY9fr9nHOw2tvG/WOL5+5bxElzXsKAhEZFg50NzOc+sreGpVOevK60kLGR+cU8TXrpjHRfMmpvRw0sdKQSAiSau+pYNNFfVs3FfPxn0NbKyoZ3dNM+4wd1IeX738ZK5aVMyEPI0bdDwUBCKSMJ1dEXZUN1F5sJWK+hbeqW+l4mArlfUt7D0QZm9dy6G2xQXZzJ9SwDWLpvChkycyr1gjhw4VBYGIxN3Wdxp5anU5z6zZR3Vj26HpIYOJ+dlMLshm4dRCPnFGCQumFLCgOJ9xual985ggKQhEJC4ONLfz7LoKnlpdzvryetJDxgVzJ3DZKZMoGTua4sJsinKzNChcAigIRCRw97y4nfte3k5HlzNvcj5fv2IeVy8q1q/8JKEgEJFAvbWnju+9uI2L503kjg/NZn5xQaJLksMEtg9mZj81syoz29jH/AIze87M1pnZJjP7TFC1iEhitHV2cddT65lSmMP3Pr5IIZCkgjwY9zBwST/zPwdsdveFwPnAd80sM8B6RCTOfvjyTnZWN/OtjyzQ4G9JLLAgcPelQF1/TYA8iw4Snhtr2xlUPSISX9v2N/LDV3Zw9aJiLjhpQqLLkX4k8vT8fcDJQAWwAbjD3SO9NTSzW81spZmtrK6ujmeNInIMIhHnrqfWMzorna9doSEfkl0ig+DDwFqgGFgE3GdmvV4h4u4PuvsSd19SVFQUzxpF5Bg8uryU1WUH+drl8xivnkFJL5FB8BngaY/aAewG5iawHhEZApX1LXznD1v5wOzx/OVpUxJdjgxCIoOgDLgQwMwmAicBuxJYj4gMUmtHF+5+xHR352u/3khnJMK3rzlF9wkeJgI7jW9mTxDtDTTezMqBbwAZAO7+APCvwMNmtgEw4MvuXhNUPSJy/Mpqw/zgT9t5es0+stNDzJqQy4lFudF/J+RS1djGi1uq+MplcykZp7uDDReBBYG7f3KA+RXAxUG9v4gMrK2zi+/+cRtv7anjA7OLuHjeROYX5x/xS778QJj7/rSDJ1eVEwoZnzxzGumhEDuqmnhjZy1Pr9l3qO2CKfn8zbknxHtV5DioY69Iitpd08zfPb6aTRUNzJucz31/2s69L21nSmEOF82byMXzJjJt7Ch+9OpOfrVyL4Zx/VklfPaCE5mY/95hnxtbO9hZ3czumibOOmGcxgsaZqy343zJbMmSJb5y5cpElyEyrD2zppyvPrORjPQQ3/noqVw8fxK1TW289HYVf9y0n9e2V9PWGe3NnZFmfPyMaXz2/BMpLsxJcOVyrMxslbsv6W2e9ghEUkhzWydf+81Gnl69jzNnjOWeTy5ickH0y31cbhbXLpnGtUumEW7vZOm2GnZUNXLN4ilMHaPj/SOZgkAkie2qbuK/39gDQGZ6iMz0EFnpaWSmhxidmcZ5c4qYPm70gK/j7qwsPcCXn1zPntpm7rhwNn//Fyf2eQhnVGY6lyyYBEwawrWRZKUgEElS5QfCXPfQcg6E28nJTKOtI0J7V4SuyHsP555WUshHFk/hilOLGTP6vcN1vf1OA8+tq+C5dZWU1YWZmJ/FYzefzTmzxsVzVSTJ6RyBSBKqaWrj2gfepKapjf932znMnfTuRfedXdFAqGls5/mNlTyzeh9b9zeSHjLOP2kCVy6cTFltmOfWV7BtfxMhg3NPHM+VpxZzySmTyM/OSOCaSaL0d45AQSCSZBpbO/jkQ8vYUdXEozedxZIZY/tt7+5sqWzkmTXl/GZtBVWxWz+eOWMsVy6czKWnTNYwD6KTxSLDRWtHF7c8spK3Kxt56FNLBgwBADNjXnE+84rncdelJ7N27wEmF+Soh48MmoJAJEl0dkW4/Yk1LNtVxz2fWMQFc49+6Oa0kHH69IHDQ6QnBYHIMagPd/DA0p388q29GDA6Kz36yExjdFY6uVnpnDFjDJedOpkJedkDvp67809Pb+CPm/fzzSvncfUiDdYm8aMgEDkKrR1dPPzGHn70yk4aWju4ZP4kxo7OpLmtk+b2LprbOjnY0sGumiZ+t6GSf/ntZs6ZNS56onbBJApHvdurp6qxlTVlB1lTdpAVu2tZXXaQ2y+czac1PIPEmU4WixC9kUq4owsDRmWmHTHWTmdXhF+tKuf7L25jf0MbF5xUxJcumcvJk3u9hQYA2/c38ty6Cp5dV8Ge2jAZacYHZhcxKjONNWUH2XewBYD0UPQY/+WnTObW82ZqxE4JhHoNyeDs3Anf/S48+ig0NUFuLtxwA3zhCzBrVqKrG5C7805DKzuqmthR1cSemmaa27to74zQ0RWhvTPa7bK9M0JLRxdNbZ3RX/JtXTS3d9L9XyEjzSjIyaRwVAYFORkU5mSwu6aZXTXNLC4p5K5L5nLWzMH3w3d3NlVE+/P/bkMl7rCopJDF0wpZXDKG+cX5ZGekBbRVRKIUBDKw3/8ePvYx6OiIPrplZEQfTz4Jl16auPp60dzWyUtvV/HK1ip2VDWxs6qJ5vauQ/PzstLJy04nMz1ERlro3X/TQuRkppGblc7orOgx/bzYMX4H6ls6OBjuoL6lnYPh6N/ZGSH+1wdncfG8ifrFLsOSuo9K/3bujIZAOHzkvO5g+NjHYP36hO8ZdH/5P7++kpe3VtHWGWHc6ExOnpzPXy2Zxqyi0YfGxi/KzdKXtsggKAhSVHNbJyv21NHRGeHUf/82E9o7+r9dXUcHfO97cN998SrxkPqWDl7ZWsXvN7xz6Mt/Ql4WnzhjGpedMpklM8aSFtIXvsixUhAMMwea29lYUU/IDDNIMyMtZJgZWekhJhVkM250Zq+/hMtqw/zp7f38aWs1y3bW0t4VHWZ4w69+Qaiz44j279HRAT//+aCDoD7cwZu7allddgB3Jys9jaz0EFkZ0UHTsjNCjM/NYuqYUUwdk8PorPd+FCvrW3hx837+uHk/b+6spTPi+vIXCYiCYBj546Z3uOvpDdQ1t/fbLjMtGgiTY4/c7HSW7apjR1UTADPHj+ZT50zngrkTyM/OIPc7rYMroKmpz1mtHV2s3HOAP++s4c87ati4r56IR0fMzAgZrZ1HDpbW05hRGYdCoeJgC+vK66O1Fo3m5g/M5OL5E1k0tZCQvvxFhpyCYBgIt3fyr7/dzBMr9jK/OJ/vf3wRWekhIg4RdyLudEWc1o4u3qlvpfLQo4W39hzgYLidxSVjuO7MEv5i7gRmjD9s2OLcXGhsHLAOz82l+2vY3dlR1cSr26p5dVs1y3fX0d4ZIT1kLC4p5PYLZ3PuieNZOLWQzPToQafOrghtndFHS0cXVQ2tlB9oiT3ClB9oYev+RgpyMvjSJSdx8bxJnDghd4i3pogcTkGQIK0dXfxiRRmPryijZOwoLl0wmQ/Nm0hBzntHhly39yB3/nIte2qbue2Ds/iHi+Yc+mIdMjfcAP/1X+/tLXSY9lAar552Ec1r9rF8dy2vbq2moj66JzF7Qi43nj2d9584njNPGHvEYZ5u6Wkh0tNCjI6NfzalMIfFJWOGdl1E5Kip+yjRX7d1ze2UH2hh38EW9nX/e7CFjDRjQl42E/OzmZifFfs7i4kF2eRlpR91r5TWji6eWFHGA6/uZH9DGwunFlDd2EZFfSsZaca5J47nsgWTufDkCTyxoozvv7idorws7r52UXBjyO/cCaee2nuvoZiO7ByuuuV+toyaQF5WOueeOJ4PnlTEeXOKmKLBzUSSnq4jiOnoilBaG472Oa9uYvv+RnZUN7Gruplwj/7nALlZ6RQXZtMZcaoa2mhq6zzi9UZnpjG5MIfJBdlMys9mcmEOk/KzKcrLOvQYn5tJVnoarR1dPLY8GgDVjW2cecJY7vzQbM6JXZi0rrye5zdU8vyGSsoPtBx6jytOncy3rzmFglEBjyE/iOsI6s67kLK6MPOL88nQzclFhhUFAfC79ZXc+cs1dHS9u77FBdmH+pyXjB3FlMIcpozJYWrhKPJz3vtrv7mtk6rGNvY3tB56VNa38k59KxX1rbxT30JVYxu9bc6CnAwi7jS2dnL2zLHcceGcPn/duzsb9zXwwpb9zJmYy+WnTI5fX/idO6NdRH/+83evLL7xRvj85xN+/YCIHB8FAdFxX55es48Ti3KZPTGXWUW5fR7LPlYdXRGqG9uoaWqjurHHo6mNcHsXHzt9KmcfxdAEIiJDRVcWA7Mn5vHlS+YG+h4ZaSGKC3VDEBEZXnSgV0QkxSkIRERSnIJARCTFKQhERFKcgkBEJMUpCEREUpyCQEQkxSkIRERS3LC7stjMqoHSfpoUAPWDnH74tIGejwdqBl3s0emr7qFarr92R7PNeps+UrfbQG2O57PW27Sez4PcZn3VM1TLBLndRupnbaB2Q/FZK3T3ol5f3d1H1AN4cLDTD582iOcr4133UC3XX7uj2WaptN0GanM8n7WBtluQ2+xYt9tQfNaOd7uN1M/aQO2G+rN2+GMkHhp67iimHz5toOdBOtb3Guxy/bU7mm3W2/SRut0GanM8n7XepiX7dhuKz1p/8/V/9OjnHetn7T2G3aGhRDKzld7HoE3SN223o6dtdmy03Y7NSNwjCNKDiS5gmNJ2O3raZsdG2+0YaI9ARCTFaY9ARCTFKQhERFJcygaBmf3UzKrMbOMxLHu6mW0wsx1mdq/1uJekmV1rZpvNbJOZPT60VSdWENvMzD5tZtVmtjb2uHnoK0+soD5rsfkfMzM3sxF3gjSgz9ttselrzex1M5s39JUPQ0H1uU32B3AecBqw8RiWXQGcAxjwe+DS2PTZwBpgTOz5hESv5zDYZp8G7kv0ug237RablwcsBZYBSxK9nsNhuwH5PdpcBfwh0euZDI+U3SNw96VAXc9pZjbLzP5gZqvM7DUzO+LelmY2meiH6U2PfpoeAa6Jzb4FuN/dD8TeoyrYtYivgLbZiBfgdvtX4DtAa4DlJ0wQ283dG3o0HQ2otwwpfGioDw8Cf+/upwP/CPywlzZTgPIez8tj0wDmAHPM7M9mtszMLgm02uRwvNsM4KNmtt7MnjSzacGVmlSOa7uZ2WJgmrv/NuhCk8xxf97M7HNmtpNoiN4eYK3DRsrcvH4gZpYLvA/4VY/DsFm9Ne1lWvevinSih4fOB6YCr5nZAnc/OLTVJoch2mbPAU+4e5uZ3Qb8N/AXQ11rMjne7WZmIeB7RA+rpYwh+rzh7vcD9xm1HvgAAAP+SURBVJvZdcBXgb8e4lKHHQXBu0LAQXdf1HOimaUBq2JPnwV+RPRLvttUoCL2dzmwzN07gN1mtpVoMLwVZOEJdNzbzN1re0x/CPg/gVWbPI53u+UBC4BXYl+Ik4Bnzewqd18ZcO2JNBT/R3v6RaxtytOhoZjYscPdZvZXABa10N273H1R7PF1d68EGs3s7FhPhE8Bv4m9zK+BC2LLjyd6qGhX/NcmPoZim8WO53a7CtgS7/WIt+Pdbu5e7+7j3X2Gu88gerJ4pIfAUH3eZvd4ycuB7fFej6SU6LPViXoATwCVQAfRX/I3AScAfwDWAZuBr/ex7BJgI7ATuI93r9A24O7YshuATyR6PYfBNvt3YFNs+ZeBuYlez+Gw3Q5r8wojs9dQEJ+3e2Kft7Wxz9v8RK9nMjw0xISISIrToSERkRSnIBARSXEKAhGRFKcgEBFJcQoCEZEUpyCQEcHMmuL8fm8M0eucb2b1ZrbGzN42s/8cxDLXaNRMGUoKApFemFm/V927+/uG8O1ec/fFwGLgCjM7d4D21wAKAhkyGmJCRiwzmwXcDxQBYeAWd3/bzK4kOsZMJlALXO/u+83sm0AxMAOoMbNtQAkwM/bv99393thrN7l7rpmdD3wTqCE67MMq4AZ3dzO7jOgFhjXAamCmu1/RV73u3mJma3l3YLlbgFtjde4AbgQWEb0C+4Nm9lXgo7HFj1jP49h0kmK0RyAjWV8jVb4OnB37Ff4L4Es9ljkduNrdr4s9nwt8GDgT+IaZZfTyPouBO4n+Sp8JnGtm2cCPiY6D/36iX9L9MrMxRMemWhqb9LS7n+HuC4kOvXGTu79BdDydL3p0SIWd/aynyKBoj0BGpAFGqpwK/DI2zlEmsLvHos+6e0uP579z9zagzcyqgIm8d4hjgBXuXh5737VE9yiagF3u3v3aTxD9dd+bD5jZeuAk4D/c/Z3Y9AVm9i2gEMgF/uco11NkUBQEMlL1OlJlzA+Au9392R6Hdro1H9a2rcffXfT+f6a3Nr0NhdyX19z9CjObA7xuZs+4+1rgYeAad19nZp8mOrz54fpbT5FB0aEhGZG8j5EqY7MLgH2xv4Mai/5tYKaZzYg9//hAC7j7NqKD8H05NikPqIwdjrq+R9PG2LyB1lNkUBQEMlKMMrPyHo9/IPrleZOZrSM64uTVsbbfJHoo5TWiJ3KHXOzw0meBP5jZ68B+oH4Qiz4AnGdmJwBfA5YDLxANlm6/AL4Y63I6i77XU2RQNPqoSEDMLNfdm2Jj4t8PbHf37yW6LpHDaY9AJDi3xE4ebyJ6OOrHCa5HpFfaIxARSXHaIxARSXEKAhGRFKcgEBFJcQoCEZEUpyAQEUlx/x/4KKSF123t1gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.lr_find()\n",
    "learner.recorder.plot(skip_end=10,suggestion=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lr = 6E-06"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>error_rate</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.709942</td>\n",
       "      <td>1.720260</td>\n",
       "      <td>0.232649</td>\n",
       "      <td>0.767351</td>\n",
       "      <td>00:19</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAf4ElEQVR4nO3deZDc5X3n8fe3j7kvzSXNITG6jA48DHhQcHBAJLEjMEh4S2ZF7BwsjmI7lNdsVTYkqbKd2KnaxKlNyrvERMlqnVQ4khCIFa+57FgohTk0AiEkJCGhcxhpbs199PHdP7olN2KOFprRSD8+r6qu6d/zPL9fP09r5tO/fvr5tczdERGR4ArNdQdERGR2KehFRAJOQS8iEnAKehGRgFPQi4gEXGSuOzCRvOIyX/mRpYTN5rorIiJXhF27dnW5e9VEdZdl0HtRFZ/95j/wB7evnOuuiIhcEczs+GR1l+XUTU44xL8f6Hhfudb8i4hcuMsy6CuKcjjcMUhr7/C5sgOn+7n+m8/zlcde52jX0Bz2TkTkyjJt0JvZVjPrMLO9k9T/rpntTt/2mlnCzMrTdcfM7M10XUu2nSrOiwKw/WAnAINjcf7gyTfpHY6x7Y02/sv3djI0Fs/2cCIiH2rZzNF/D/jfwN9PVOnu3wa+DWBmdwIPuHtPRpNb3b3rQjqVGwlRVp7Pc2+18/kbr+LRV47z2okzfHtjIwvLC/jVv3mZrzz2Or99y1KS7ty4pOLcvvva+ijOjbKoouBCHlJErmCxWIzW1lZGR0fnuiuzLi8vj/r6eqLRaNb7TBv07r7DzBqyPN49wGNZP/oU7mqq46GfHKa9f5Tn9rWzsqaEzzYvBOCP1q/ma9v28eP0PP6X1i6lojCHR145wdGuIUIGdzTWcuOSCg53DNI1OMZXfmkZy6qLZ6JrMyqZdEKh964uGh6P88rRHhIJZ3VdCTWl+XPUO5ErQ2trK8XFxTQ0NGABXq3n7nR3d9Pa2srixYuz3s+y+YAzHfQ/cPdrpmhTALQCy86e0ZvZUaAXcOCv3X1LNp1qbm72J555gbV/vp1F5QWc6BnmD29fyW/dvORcm2NdQ7x8pJs/f+5tugbHACjJi/Dbtyyld2icR189wfB4gmjYCJmRFw3zSyuqyY2GiYaN+nn5FOZGKMqNsKqmhGXVRe/5BYknkjzyygm6h8a5blEZ84vzcJxVNSUT/iK9dqKXQ+0DAAyPJ2i+qpx/ePk4/7anjeK8CIkkrKotYf+pfhaU5BEJG33DMY73DDOvIEos4fzSimq6h8Z54e3Oc8eNho1VNSXUzcunODdKQ2UhiysLyIuGKS/M4fu723i3d4T5JbncdV0dZ4ZjdA6O8cmV8xkcizO/JI9YIklh7mW5wEpkRuzfv58VK1YEOuTPcncOHDjAypXvXZVoZrvcvXmifWbyr/9O4MXzpm1ucvc2M6sGnjezA+6+Y6KdzWwzsBlg0aJFNFQWsqahnFeP9bBmcTn3feK9r14NlYU0VBZyd/NCHDjaNURVUS6lBam3M/f/4jJae0dYWlXE6f5R7v7rl3jy9XcBKMwJMzSeeM/xrp5fzH2fWIzjvHKkh4PtA+xr639fP1csKKZ+Xj5Lq4sYHU/wgz2nKM2PcmSCD4gjIWP9tbXEkk7SnYOnB/hoXSlj8QTj8SRLq4v4xPJKBsfiuMMLb3dSkBvm0401XLewjNW1pfy/N9s41D7Ij/d3UJgboWdofMInP2Twdy9NurqKFQuKqS3LZ3l1EXc01vLjA+2MjCdorC/jo3WltPWNUF2cy3giyYuHu3npnW7KCqJUF+dSXZzLR+tLqSjMZWF5AeHQxH9MA6Mx2s6MsrSqEDM7187dGYsneetUP8NjCVbWFHOqb5QTPcMcah+k5XgPOeEQkbCx/9QA1y0q44aGco53D3Gse5iy/CixRJKkw80fqaJpYdm5x5Ar18h4gv7RGPNL8mbkeB+W34cPMs6ZPKN/Cvhnd390kvpvAIPu/ufTPV5zc7O3tLRwonuYv/zR22y+ZQkrFpRM28+p9I/GONE9TF1ZPvMKc+gaHOPUmVEGx+Ic7hjgkVdOcOB06ow8JxxidV0J66+t5TPX1XHw9ABHu4YYiyf5m/84QjQc4mTPMPGk07SwjJL8KL+8spo1i8spyo3gDtvf7uTnl1awtKroovp9voHRGMe7hxmNJTjYPkBNaR6/uGI+fSMxnn+rneriXHIjIX76TjeVRTl0DY4TTybZebSXgbE4h9oHiCen/zdfXFnIaCxB58DY+9qXFUQJm9FQWci61Qs41j3EruO9vNM5SCzh557D6pJcbmgo59WjPbx7ZmTSx7qqooDOgTFyIyGWVBVxqH2A/tHUh+2VRbnkhI1oJMSZ4Rh9IzEA5hVE+dhV5dSW5XGiZ5jC3AivHe9l4bzUO8DivAhfvnUphtE3EqOiKIehsTjxpNNYV0ZVcS6vHO3mpmWVDI7GeaP1DFdVFPKR+UUU5Hzw85+hsTg/2t9OaX6UaDhEXjTE6tpSTvWNcmZ4nGg4xPL5ReRGwlkf81D7AKOxJI7z6tEeCnMjDIzG6BwYY+exXkrzo1QV53KoY5AllYVUl+RSX5bPWDyJO5TkRzAzakrzWFCSx/L5szuFmUg6f/bMAQ62D7CksojNNy9hQWke7s62N9p4dt9phsYSvHail4HROGsayvnNmxpYVl1E0p1jXUP0j8T5+WUV1JXlTxhsyaQTSybPPY/79+9/3xkupP5e+kZiJJJOfk6YRNIpyo2QGwkzMBpjJJZgZDyBmRENG0W5EaLhEA7gTjQSIp5InaiZpRaKhKYI2rF46njRcIiCnPCsvfhMNN6pzuhnJOjNrBQ4Cix096F0WSEQcveB9P3ngT9292eme7yzQX8pJZPOvx/oIJ50Pr6k4tw7g8kc7RriePcQv7C8atIz3MtRx0DqM49r68tYPr+I7Qc76RgYZUFJHvtPDZBwZ0NT7bkXqGTS6Rwc49WjPfSNxDjWNcSp/lHeOHmGwbE4Z4ZTwVtXls/6ploqi3IZHI3TPxpj+8EOuofGWdNQzuraUpZVF1FWEGXvu30sKM1jeXUxNaV5zCvMeU8fE0nndP8oJXmRcyuwIPVHtP/UAG+fHmDnsR5ajvfSdmaE3EiI/tE419SVkBsJs6i8gN0nz3ygZbhmsHBeAbdds4A7Gmv56x3vcLJnmAc++RGqinPZ09rHyHiCeDLJk6+9S140TF40xOLKIlbXlvDYqyfe907QDDL/zApywly3qIzB0dTU2tqrqykvjJJ0WFRewNBYPD3lF+WdjkGee6t90v6ePc6RriHqyvI53T/KeDw55fiaFpbRPxLjjsZafmF5JbGEU5of5SPzi4iEJ16I1zM0ztGuIa5eUExRehowmUyFH8Duk2do7x9lT2sfL7zdyb62flYsKOZI5xBFeRG+ueEadh3vZeuLR6ktzWNgNE59eQG/sno+/7jzJKf6Jv4Q9er5xSwsz2dfWz/LqosYGI0zGkvwbu8IY4kkn1o1n9W1pVxTMEDlwiXkR8PkRELn3kUOjsUxMyIhI5Z4//NydlrX3YklnfgEbd7z/GGEQlCaHyUSDhEyGBpLcKqji6f/9Qk+++v3nWsbNiMSDpGfEyYSSr3DzUm/AIwlkiQSzt3/aT2PPvIoVZXlACSSScbiSRJJJ5ZIEgml2p//7zLjQW9mjwFrgUqgHfg6EAVw94fTbX4TWOfumzL2WwI8ld6MAI+6+59M+WBpcxH0cuHiiSQdA2NUFecSCdmcvHV2d9xT79jKCn72gjEaS7D/VD8l+VGK8yK8cbKPquJc5hVEee1EL609IyypKqLtzAiRsHFDQznvnhnh4OkB9rT28eMD7bhDOGSU5kcnnDJbUlVISV6U/pEY7f2jDI0niISMb951Dcuri0gknf7ROLtP9lI/r4CqolxG4wlePNzFm++mVocd7x6ibZKQCxmU5Ef5zHV1NNaXkkjCLR+pYng8TtJT9VdVFJ57Hs4+/2eGx4knPRV4SRgYizEaS3Lw9AAtx3t46Z1uxhNJjnYNvecFqKwgym/fvJTxeJLXT6beHY3HkwyNx3lm72niydQLwrLqIk6dGaF9YIy8SCrIugZ/9vysWFDMxo/V84VfWMLhjkF+8/++Smtv6h3d+mtr+Yv/3EQi6YQMIuEQ4/Ekr53o5VTfCDnhMAtKc8mLhvnp4W62v91Ba+8IeZEwXYNjrKhJvRspyIlQXZzL03tP0zM0zta7ali6/GoG0u8EQ+mpw7KCKJVFuYQs9TvhQCyRCtH8aJj8aPjcYgh3P/fOyYCkw3g8SU4khAHjiSSjsQRj8eS5d5aQCv++znf5wq9+lu0v76IkP8JYLMnAWJzxWJxYkglfZDJFwyES6Sne84VDRl76nctYPEk4ZJw6fpi/2Jk60VxUXkB+Tpj1TXUXd0Z/qSnoZa4d7hjgUPsgy+cXUVuWz7bdbRTkRmiqL6MkP8J4IklVUe65cE0knbYzIxTkhKkoys36cZJJ53DnIJ0DY+TnhGk7M0L/SJxPN9ZQkheZ1RfP/af6ebt9gGg4xNBYnB/sOXVuIUBN+qy7MDdMLOHc0DCPu5rq+Lc9bfQMjVNbms+C0jxGY0nODI9z07JKVtaUUFmUQ/V5c+4j4wleP9lLa+8IdzXVkROZues0E0lnJJbgxDtvs2rVqhk77nTiiSShkBFPJImGQ9xzzz18//vf5+qrryYajVJUVERNTQ27d+9m37593PWZz3DyxElGR0f5whe/zH1f+C3CIWP11ct59oUX6ert4zfu/gzNP/dx9rz2KjW1dfzjPz9JXn4ePUPjxBOOA3mREAl3Th45xLde7Odg+8C5F+vjf3qHgl5EpubuvHSkm2g4xA0N5XPdnQuSOZXxR/+2j7cmWEhxMVbVlvD1O1dPWn/s2DHuuOMO9u7dy/bt2/n0pz/N3r17zy2B7Onpoby8nJGREW644QZeeOEFKioqaGhooKWlhcHBQZYtW0ZLSwtNTU3cfffdrF+/ns9//vNTjjeWSHK6L/VZ46ra0kuy6kZErmBmxs8vrZzrbgTCmjVr3rPO/Tvf+Q5PPZWayT558iSHDh2ioqLiPfssXryYpqYmAD72sY9x7NixaR8nGg6xsHz6i0MV9CISKFOdeV8qhYWF5+5v376dH/3oR7z00ksUFBSwdu3aCa/gzc392ZRfOBxmZGTylWoX6rL8UjMRkStJcXExAwMDE9b19fUxb948CgoKOHDgAC+//PIl7p3O6EVELlpFRQU33XQT11xzDfn5+cyfP/9c3bp163j44YdpbGzk6quv5sYbb7zk/dOHsSJyxZvsgqmgutB19Jq6EREJOAW9iEjAKehFRAJOQS8iEnAKehGRgFPQi4gEnIJeRGQOFBWlvgq8ra2NjRs3Tthm7dq1zMRScwW9iMgcqq2t5YknnpjVx9CVsSIiM+D3fu/3uOqqq/jyl78MwDe+8Q3MjB07dtDb20ssFuNb3/oWGzZseM9+md98OTIywr333stbb73FypUrZ+z7bhT0IhIsTz8Ip9+c2WMu+Cjc9j+mbLJp0ya++tWvngv6f/qnf+KZZ57hgQceoKSkhK6uLm688UbWr18/6f8z8N3vfpeCggL27NnDnj17uP7662ek+wp6EZEZcN1119HR0UFbWxudnZ3MmzePmpoaHnjgAXbs2EEoFOLdd9+lvb2dBQsWTHiMHTt28JWvfAWAxsZGGhsbZ6RvCnoRCZZpzrxn08aNG3niiSc4ffo0mzZt4pFHHqGzs5Ndu3YRjUZpaGiY8CuKM83G/yqmD2NFRGbIpk2bePzxx3niiSfYuHEjfX19VFdXE41G+clPfsLx48en3P/mm2/mkUceAWDv3r3s2bNnRvqlM3oRkRmyevVqBgYGqKuro6amhs997nPceeedNDc309TUxIoVK6bc/0tf+hL33nsvjY2NNDU1sWbNmhnpl76mWESuePqa4ov8mmIz22pmHWa2d5L6tWbWZ2a707evZdStM7ODZnbYzB68wLGIiMgMyGaO/nvAumna/Ie7N6VvfwxgZmHgIeA2YBVwj5mtupjOiojIhZs26N19B9DzAY69Bjjs7kfcfRx4HNgwzT4iIh/I5TgNPRs+yDhnatXNx83sDTN72szO/hfsdcDJjDat6TIRkRmVl5dHd3d34MPe3enu7iYvL++C9puJVTevAVe5+6CZ3Q78K7AcmGgx6KT/Cma2GdgMsGjRohnoloh8WNTX19Pa2kpnZ+dcd2XW5eXlUV9ff0H7XHTQu3t/xv0fmtlfmVklqTP4hRlN64G2KY6zBdgCqVU3F9svEfnwiEajLF68eK67cdm66KkbM1tg6Uu5zGxN+pjdwE5guZktNrMcYBOw7WIfT0RELsy0Z/Rm9hiwFqg0s1bg60AUwN0fBjYCXzKzODACbPLURFnczO4HngXCwFZ33zcroxARkUnpgikRkQC4qAumRETkyqagFxEJOAW9iEjAKehFRAJOQS8iEnAKehGRgFPQi4gEnIJeRCTgFPQiIgGnoBcRCTgFvYhIwCnoRUQCTkEvIhJwCnoRkYBT0IuIBJyCXkQk4BT0IiIBp6AXEQk4Bb2ISMAp6EVEAk5BLyIScNMGvZltNbMOM9s7Sf3nzGxP+vZTM7s2o+6Ymb1pZrvNrGUmOy4iItnJ5oz+e8C6KeqPAre4eyPwTWDLefW3unuTuzd/sC6KiMjFiEzXwN13mFnDFPU/zdh8Gai/+G6JiMhMmek5+vuApzO2HXjOzHaZ2eYZfiwREcnCtGf02TKzW0kF/Scyim9y9zYzqwaeN7MD7r5jkv03A5sBFi1aNFPdEhH50JuRM3ozawT+Ftjg7t1ny929Lf2zA3gKWDPZMdx9i7s3u3tzVVXVTHRLRESYgaA3s0XAk8CvufvbGeWFZlZ89j7wKWDClTsiIjJ7pp26MbPHgLVApZm1Al8HogDu/jDwNaAC+CszA4inV9jMB55Kl0WAR939mVkYg4iITCGbVTf3TFP/BeALE5QfAa59/x4iInIp6cpYEZGAU9CLiAScgl5EJOAU9CIiAaegFxEJOAW9iEjAKehFRAJOQS8iEnAKehGRgFPQi4gEnIJeRCTgFPQiIgGnoBcRCTgFvYhIwCnoRUQCTkEvIhJwCnoRkYBT0IuIBJyCXkQk4BT0IiIBp6AXEQk4Bb2ISMBNG/RmttXMOsxs7yT1ZmbfMbPDZrbHzK7PqFtnZgfTdQ/OZMdFRCQ72ZzRfw9YN0X9bcDy9G0z8F0AMwsDD6XrVwH3mNmqi+msiIhcuGmD3t13AD1TNNkA/L2nvAyUmVkNsAY47O5H3H0ceDzdVkRELqGZmKOvA05mbLemyyYrn5CZbTazFjNr6ezsnIFuiYgIzEzQ2wRlPkX5hNx9i7s3u3tzVVXVDHRLREQAIjNwjFZgYcZ2PdAG5ExSLiIil9BMnNFvA349vfrmRqDP3U8BO4HlZrbYzHKATem2IiJyCU17Rm9mjwFrgUozawW+DkQB3P1h4IfA7cBhYBi4N10XN7P7gWeBMLDV3ffNwhhERGQK0wa9u98zTb0DvzNJ3Q9JvRCIiMgc0ZWxIiIBp6AXEQk4Bb2ISMAp6EVEAk5BLyIScAp6EZGAU9CLiAScgl5EJOAU9CIiAaegFxEJOAW9iEjAKehFRAJOQS8iEnAKehGRgFPQi4gEnIJeRCTgFPQiIgGnoBcRCTgFvYhIwCnoRUQCTkEvIhJwWQW9ma0zs4NmdtjMHpyg/nfNbHf6ttfMEmZWnq47ZmZvputaZnoAIiIytch0DcwsDDwEfBJoBXaa2TZ3f+tsG3f/NvDtdPs7gQfcvSfjMLe6e9eM9lxERLKSzRn9GuCwux9x93HgcWDDFO3vAR6bic6JiMjFyybo64CTGdut6bL3MbMCYB3wLxnFDjxnZrvMbPNkD2Jmm82sxcxaOjs7s+iWiIhkI5ugtwnKfJK2dwIvnjdtc5O7Xw/cBvyOmd080Y7uvsXdm929uaqqKotuiYhINrIJ+lZgYcZ2PdA2SdtNnDdt4+5t6Z8dwFOkpoJEROQSySbodwLLzWyxmeWQCvNt5zcys1LgFuD7GWWFZlZ89j7wKWDvTHRcRESyM+2qG3ePm9n9wLNAGNjq7vvM7Ivp+ofTTT8DPOfuQxm7zweeMrOzj/Wouz8zkwMQEZGpmftk0+1zp7m52VtatOReRCRbZrbL3ZsnqtOVsSIiAaegFxEJOAW9iEjAKehFRAJOQS8iEnAKehGRgFPQi4gEnIJeRCTgFPQiIgGnoBcRCTgFvYhIwCnoRUQCTkEvIhJwCnoRkYBT0IuIBJyCXkQk4BT0IiIBp6AXEQk4Bb2ISMAp6EVEAk5BLyIScFkFvZmtM7ODZnbYzB6coH6tmfWZ2e707WvZ7isiIrMrMl0DMwsDDwGfBFqBnWa2zd3fOq/pf7j7HR9wXxERmSXZnNGvAQ67+xF3HwceBzZkefyL2VdERGZANkFfB5zM2G5Nl53v42b2hpk9bWarL3BfzGyzmbWYWUtnZ2cW3RIRkWxkE/Q2QZmft/0acJW7Xwv8L+BfL2DfVKH7FndvdvfmqqqqLLolIiLZyCboW4GFGdv1QFtmA3fvd/fB9P0fAlEzq8xmXxERmV3ZBP1OYLmZLTazHGATsC2zgZktMDNL31+TPm53NvuKiMjsmnbVjbvHzex+4FkgDGx1931m9sV0/cPARuBLZhYHRoBN7u7AhPvO0lhERGQClsrjy0tzc7O3tLTMdTdERK4YZrbL3ZsnqtOVsSIiAaegFxEJOAW9iEjAKehFRAJOQS8iEnAKehGRgFPQi4gEnIJeRCTgFPQiIgGnoBcRCTgFvYhIwCnoRUQCTkEvIhJwCnoRkYBT0IuIBJyCXkQk4BT0IiIBp6AXEQk4Bb2ISMAp6EVEAk5BLyIScFkFvZmtM7ODZnbYzB6coP5zZrYnffupmV2bUXfMzN40s91m1jKTnRcRkelFpmtgZmHgIeCTQCuw08y2uftbGc2OAre4e6+Z3QZsAX4uo/5Wd++awX6LiEiWsjmjXwMcdvcj7j4OPA5syGzg7j9199705stA/cx2U0REPqhsgr4OOJmx3Zoum8x9wNMZ2w48Z2a7zGzzZDuZ2WYzazGzls7Oziy6JSIi2Zh26gawCcp8woZmt5IK+k9kFN/k7m1mVg08b2YH3H3H+w7ovoXUlA/Nzc0THl9ERC5cNmf0rcDCjO16oO38RmbWCPwtsMHdu8+Wu3tb+mcH8BSpqSAREblEsgn6ncByM1tsZjnAJmBbZgMzWwQ8Cfyau7+dUV5oZsVn7wOfAvbOVOdFRGR6007duHvczO4HngXCwFZ332dmX0zXPwx8DagA/srMAOLu3gzMB55Kl0WAR939mVkZiYiITMjcL7/p8ObmZm9p0ZJ7EZFsmdmu9An2++jKWBGRgFPQi4gEnIJeRCTgFPQiIgGnoBcRCTgFvYhIwCnoRUQCTkEvIhJwCnoRkYBT0IuIBJyCXkQk4BT0IiIBp6AXEQk4Bb2ISMAp6EVEAk5BLyIScAp6EZGAU9CLiAScgl5EJOAU9CIiAaegFxEJuKyC3szWmdlBMztsZg9OUG9m9p10/R4zuz7bfUVEZHZNG/RmFgYeAm4DVgH3mNmq85rdBixP3zYD372AfUVEZBZlc0a/Bjjs7kfcfRx4HNhwXpsNwN97ystAmZnVZLmviIjMokgWbeqAkxnbrcDPZdGmLst9ATCzzaTeDQCMmdneLPp2pasEuua6E5eAxhksH5ZxwpU11qsmq8gm6G2CMs+yTTb7pgrdtwBbAMysxd2bs+jbFU3jDBaNM3iCMtZsgr4VWJixXQ+0ZdkmJ4t9RURkFmUzR78TWG5mi80sB9gEbDuvzTbg19Orb24E+tz9VJb7iojILJr2jN7d42Z2P/AsEAa2uvs+M/tiuv5h4IfA7cBhYBi4d6p9s+jXlg8ymCuQxhksGmfwBGKs5j7hlLmIiASErowVEQk4Bb2ISMBdVkEfpK9LMLOtZtaReT2AmZWb2fNmdij9c15G3e+nx33QzH5lbnp94cxsoZn9xMz2m9k+M/uv6fJAjdXM8szsVTN7Iz3OP0qXB2qcZ5lZ2MxeN7MfpLeDOs5jZvamme02s5Z0WfDG6u6XxY3Uh7XvAEtILct8A1g11/26iPHcDFwP7M0o+zPgwfT9B4E/Td9flR5vLrA4/TyE53oMWY6zBrg+fb8YeDs9nkCNldQ1IUXp+1HgFeDGoI0zY7z/DXgU+EF6O6jjPAZUnlcWuLFeTmf0gfq6BHffAfScV7wB+Lv0/b8D7soof9zdx9z9KKnVS2suSUcvkrufcvfX0vcHgP2krogO1Fg9ZTC9GU3fnICNE8DM6oFPA3+bURy4cU4hcGO9nIJ+sq9RCJL5nrq+gPTP6nR5IMZuZg3AdaTOdgM31vR0xm6gA3je3QM5TuAvgf8OJDPKgjhOSL1YP2dmu9JfwwIBHGs2V8ZeKll/XUIAXfFjN7Mi4F+Ar7p7v9lEQ0o1naDsihiruyeAJjMrA54ys2umaH5FjtPM7gA63H2Xma3NZpcJyi77cWa4yd3bzKwaeN7MDkzR9ood6+V0Rp/NVy1c6drT3+pJ+mdHuvyKHruZRUmF/CPu/mS6OJBjBXD3M8B2YB3BG+dNwHozO0Zq+vQXzewfCN44AXD3tvTPDuApUlMxgRvr5RT0H4avS9gG/Eb6/m8A388o32RmuWa2mNT3+r86B/27YJY6df8/wH53/58ZVYEaq5lVpc/kMbN84JeBAwRsnO7+++5e7+4NpP4G/93dP0/AxglgZoVmVnz2PvApYC8BHOucfxp83qfdt5NatfEO8Idz3Z+LHMtjwCkgRupM4D6gAvgxcCj9szyj/R+mx30QuG2u+38B4/wEqbeve4Dd6dvtQRsr0Ai8nh7nXuBr6fJAjfO8Ma/lZ6tuAjdOUiv83kjf9p3NnCCOVV+BICIScJfT1I2IiMwCBb2ISMAp6EVEAk5BLyIScAp6EZGAU9CLiAScgl5EJOD+P4KaCg8rWoAUAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.fit_one_cycle(1, max_lr=slice(lr*0.95**num_groups, lr), moms=(0.8, 0.9))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner.save('second_cycle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "seed_all(seed)\n",
    "learner.load('second_cycle');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner.freeze_to(-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>error_rate</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.711365</td>\n",
       "      <td>1.710787</td>\n",
       "      <td>0.242424</td>\n",
       "      <td>0.757576</td>\n",
       "      <td>00:24</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3daZRc5X3n8e+/lu7qXb1J6lZraYSEFmiEaIS8gWwcWzKL4oSxRSDEjHMUcGzHnheBnEzscPJixpOTHIeJsY6GYXBijGzjBeyDcYwDKDHIqAVCaAFJaOtWS+pF6m6p11r+86IKuSX1Ug0lNbr6fc7p03XvferW/+nlV089de8tc3dEROTiF5rsAkREJDcU6CIiAaFAFxEJCAW6iEhAKNBFRAIiMlkPXFha7smiKuZPKyE/oucVEZFsbNmypcPdq0faNmmBHi2bxtQ7/4GH7rmOj14xdbLKEBG5qJjZwdG2TdrQ2Ekf//5n/7qFrr6hySpDRCQwJi3Qk6l0oA8lUqx7cd9klSEiEhiTFuiJ1O/OUP3pa4fp7o9PVikiIoEwaXPoAA/etpjFtaV8dv0m/vtPt/O/77hmMssRkfe5eDxOS0sLAwMDk13KeReLxairqyMajWZ9n0kN9GmlMRrnVPDnH72ch369hy997HLmTyvJ+v7dfXGK8sNEwjpKJohO9A7R1R9nemmMeCpFaSz7P+yRHO0ewCz9dzeSd65rZGbv6XHk/GlpaaGkpIQ5c+YE+vfk7nR2dtLS0kJ9fX3W95vUQK8szgPg7g/M5uHn9/KjV1v4q1ULz2nn7uzv6GVmRSHRTHj/03N7+Kdf7yYWDVM7pYCv3bKIG+aPeCRP4AwmkvxmbwdXTC9lxpSCyS5nXP1DSQYTSYrzI2M++bo7ZkbPQJyfvHqY//GLXQzEUwBEw8bUkhg3zK/mL26aR3EsQnH+mX++fUMJHvmP/fzrpoPUVxVxzawpzJhSQNOBE2w5eILDXf3kRUJ8YtE0bpxfze8tmkYoZDy38xjf++0hDnT2MphIsbi2lCPdA3ScHOQj86q59eparp1dzvSyGB2nBjnSNcDhrn7CIWN6aYykO1fXlQU6YN4vBgYGAh/mkB5UVFZW0t7ePrH7TdbVFvNr5vnOba8xt7oYgD/9zma2tXTz0gMfO/1Pn0im+OZze/jhlmaO9QxSGotQWhDlcFc/7nDTgqnMKC/g+bfaaDnRz8cXTuOPl8/mI/OqTv/CN+3r5EBHL8vqK7isupiuviEG4imK8sO8/HYnH5lXTUFe+HRd/7mngxfeauODl1dSX5Vuf7Czj6FkirfbTmFmHDrey2/3HefO5bO5YV4VV84oIxYNn9PHtp4B2k4O8vybbVQW51NXXsCC6SVMHWWEOJrm431875VDvPBWO8X5YZqP93O0Z4DCvDCfvmYG00pjxJMpFtaUUpQfIZ5IccX0EmZMKSAUOvMP3935jz0d5EdCLJ5RxtZDXbyyv5OX93Vy+dQS2k8OEE86y+oruGpGGUX5ERZML6EoE57He4fYc+wk21q6+cDcSmLRMFMKo3T1xXnmjSPMrizk1oZaUu5s3NPOE6808+9vtpFMOaWxCB+eV0VD3RQG4ymGkknKC/OIRcO8ebSHJ7e0MLUkxrGeAQYTKT58eRWNc8rZ03aK8sIozcf7eentDuLJ9N/sR6+oJhwytrV0k0w5pwYTDCZS3Di/mvaTg+w80gNAcX6Exjnl1JTFGIineHb7UfrjyTN+LvOnFdNQNwUD9rSdoq68gKK8CL/ceZSuvjjF+RHmTi3m9eauEX9HJbEIKxdP567ls2lQuJ83u3btYuHCcwd9QTVSf81si7s3jtR+UgP92Ns7mFKYHqU/u/0o9353C42zyxlMpLhyRhlPvHLonPsV5YVZdVUNsyoKuW/FXKLhEH1DCb71/F6+v7mZjlNDmMHtS+s40RfnuV3HACiIhvmHz1zNN559kyPdA1QX53O4qx+AD86tpDAvQsuJPnYfO0lqjB9JOGSZYEjScSp9uGVZQZSrZ04B4EBHL3Orizg1mGDLwRPn7Cs/EmJxbSk1ZQW0dPXT1TfErIpC5k0t4XhvOoSWX1bJ4tpS5lYX48DnH9tMz0CCK6aV0BdPMKeyiFsbavnZtlbeONxNV9/IbyiXxCI01JUxu7KI2RWFNB08wfNvtp3xhvTwttFwiKkl+SRSzt62U6e3FedHWHXldJIp55ntR06PmkcTi4ZOt6kqzmP1khlMLcnn7fZTbNzdwdGe9PynGbzz52fG6SeCsoIoNy2cyor5U895Qmo+3sdTWw/TfnKQH25poW8oSW1ZjMUzyphdUciqq6Zz7ewKAOLJFK8ePMGCmlLKCn43XTOUSPGrncdo7eonnkqRHwnzuQ/OIRw6N4QH4ulXQ9/ddJCDnX3ctqSWhTWlTC9NP/G0nOgnGgnx2qETPL21lUTKWVRTys0NNXxgbiXxRIprZpWTl+OT5xLJ1CU51fjq69uZNusyCvLCxCJhYnlhQgF98nR3du3axaJFiwDoODXIK/uPc3ND7fsv0GM187zv8O7T/7DJlPN7//gi+zp6qS2L0dqd/qf/w6V1fOMPr2IomeKffr2HNdfNor6qaMR9DiVSfO+3B9l88AT/tuMo+ZEw962Yy43zq7nnsc20nxwkZFBTlp6mWL2klu7+OD/c0sJQIsXc6iI+tmAqd39gDoe7+tnZ2kN1ST6zKwspzo8wvSxGXjh0+h+p+Xgfbx49ydOvt7K37RTJVIrpZQVs3N3OwppSPji3ksunFrOsvoKBeJIDHX08/1Ybu4+dpK1nkBnlBUwvi3Ggo5c3j56kOD/CoppSNu3vZPivpawgyr/812WnnzRG6nfI4Fc7jxHLC1Mai7LrSA/f39xMd3+cE31DnBxIUFmUx41XVDOnsojZlYW83d5LZVEeNy2cSl154en9ufvpVwHHe4f4t51H+dWOYxTlR5g/vYTbr63jytpSthw8AUBr1wC1U2LMmFJAZ+8Qz7/VRkVhHktnl/PxhdPOCDN35+RgIv1zDBlvt/fSH09SUZjHrMrCc/o2lt7BBAPxJJXF+RO63/lysLOXDZubeW7nMfYMe0IsiIb5g6UzuPfGucSTKU4OJCjMC1Ndkn96QDOa5uN9vLi7nQ2bD1FfVcwnFk2jbyjBgz/biTvMn17CzPICOk4Nsq+9l9KCKO7O7dfOpHcwQcrTr7bqygtJuTOropCegTidp4Yozo/wn3s7OHyin1uvrmXe1GKO9w3R1jPIUDJFSSxCWUGUqnF+vu7O9zc3s7W5i4a6KdRMiVFfWcScUf5PJ2rTvk62tXTxw6YW/nJ5CdNmXXZ6WywSprwoj8FEkpAZhXlhTg0mCIeMaDhEyCCZSj8BlhRECZuRciccMsJmRCMh3J2BeIr+eIJUCsoK00/+/fEknnIK8sLkRcKk3DnWfpzvb/ged3/+z0ilnN6hBImkkx8NkRcJUVmURziU/nvvG0owmBnYfPYPVvN//t93KCgpJZF0ygvzKMhL73MwnuTUYPL01KQDkZCRcti/dzf3/vwoBdF0vwAOfuOW91+g18xd7Efe3nHGuubjfRzrGeDa2eX8594OqorzWVhT+q723z+UJBSC/Eh6KuTt9lP84o0j3LFsFpXF+afnawFSKccsd2+GDcSTI07BjKW7L05+NEQsGuZIdz9HutOjv4F4kk8unn7GCHOiEpkQmVIY1VTABXKos4+X3u4gFg3zm70d/OjVlnNerYUMls4qZ2FNKdtaujjSPUB+NMQV00o5lplSe625i6FEirKCKJGQ0dmbflVYXhjlxvnV7O/so6tviHDImFVRSHd/nO7+OPvae08/xlivON8RDhllBVGO9557kl9tWYy7PjCby6uLOdozkJleqqEgL8wzbxzhm8/tZvexU2e8MgOYN7WYcKbmZMq5YV4V86eXsKO1BxzePNpDfVUxKXeOdg9wrGeAJTOnMLU0n/xImAXTS+g4Ncg3n9tDIuWUF0ZZd8t0rmlYzFAiRTyVvl88mSIcMlKp9AmL7/yND882w06fzDhcYV6YeNKJJ8d+1ZkfCRNPpmg+dJAvfe6z/PjXL6d/bmZEwsbAUAILhYiEQumgTjl98SRj5Wt+JEzSnUTmsUNmFETDhEPGUDKFO3Qe3scLx/I4NZhgZnkh119WwbWzK95/gd7Y2OhNTU2T8tgiF9r2w9283tJFUV6EovwIPf1xDnT28vxbbWw/nJ7rn1aaT31VEfvae5ldWcjJgQT1VUV86WPzmD+tGDNja3MX0bCxqKZ01CkXd6e7P04kM0J99WAXrV39YNB+cpCivDCJlBMJGR+eV0VFUT6P/Mc+Wrv6WVRbSu2UAgrzwrR2DdDVN8TGPR28sv/4GY9Rkh+hYWYZv9nbyYLpJdy1fDZ3Xj+L/R29tJ8cZEdrD89uP0ooBPVVxcSTKZ554wh9Q0kqivKIRUKEw0YkFOLkQIKq4jwW1ZaytbmL7r44fUPJ0+9zXDennAdWLWBxbRn79+4+Y045lXKSnu7LUCJF+6lBppbEiIaNRMpPn8AYCRvdffF08LsDRiKZomcgQTRslMSiFOaFSaac/niSVMrJj4QIhYyBeIqTA3HyIiG+vPZz/OLnP2Pe/PnkRfMoKSmmpqaGrVu30vTaNn7/05/mcEszg4OD3LP2Pr74hfvIj4S44vK5vPiblxka6GP1rbfQeP0H+O2ml5k2vZYNT/6IirISCkYYBF40c+gKdJG0rr4hSmJRQjl8lZhrzcf7ONozQFVxPm09A2zY3Myrh07w8YXTuH/lgqzeIxiIJ9l5pIdFNaXjvoJNppyDnb1UFOWdMS01POAe/NkOdrb2vLeOnWVRbSlfv3XxqNsPHDjALbfcwvbt23nhhRe4+eab2b59++lDC48fP05FRQX9/f1cd911vPjii1RWVjJnzhyampo4deoUl19+OU1NTSxZsoTPfOYz3Hbbbdx1110jPt5EA31SD1sUEcadR38/mFlRyMyK9Hsc9VVFXH9Z5YT3EYuGWTqrPKu24ZBxWeYIuPezZcuWnXGc+EMPPcRPfvITAJqbm9mzZw+VlWf+rOrr61myZAkA1157LQcOHMhZPQp0EbkojTWSvlCKin73xu8LL7zAc889x8svv0xhYSErVqwY8YzW/PzfvckcDofp7+/PWT2X3nFPIiLvUklJCSdPnhxxW3d3N+Xl5RQWFvLmm2+yadOmC1ydRugiIlmrrKzkQx/6EFdeeSUFBQVMmzbt9LaVK1eybt06GhoauOKKK1i+fPkFr09viorIRUNnio79pqimXEREAkKBLiISEOMGupk9amZtZrZ9jDYrzGyrme0wsxdzW6KIiGQjmxH6Y8DK0Taa2RTgYeA2d18M/JfclCYiIhMxbqC7+0bg+BhN/gj4sbsfyrRvy1FtIiIyAbmYQ58PlJvZC2a2xczuHq2hma01syYza5rohdtFRGRsuQj0CHAtcDPwSeBvzGz+SA3dfb27N7p7Y3X1pfHpQiJy6SouTl++oLW1ldtvv33ENitWrCBXh3Dn4sSiFqDD3XuBXjPbCFwN7M7BvkVELnq1tbU8+eST5/1xcjFCfwr4iJlFzKwQuB7YlYP9ioi8r9x///08/PDDp5f/9m//lgcffJCbbrqJpUuXctVVV/HUU0+dc78DBw5w5ZVXAtDf38+aNWtoaGjgs5/9bE6v5TLuCN3MngBWAFVm1gJ8HYgCuPs6d99lZs8C24AU8Ii7j3qIo4hITvziATj6Rm73Of0qWPU/R928Zs0avvKVr/CFL3wBgB/84Ac8++yzfPWrX6W0tJSOjg6WL1/ObbfdNuqlkL/97W9TWFjItm3b2LZtG0uXLs1Z+eMGurvfkUWbvwf+PicViYi8T11zzTW0tbXR2tpKe3s75eXl1NTU8NWvfpWNGzcSCoU4fPgwx44dY/r06SPuY+PGjXz5y18GoKGhgYaGhpzVp4tzicjFaYyR9Pl0++238+STT3L06FHWrFnD448/Tnt7O1u2bCEajTJnzpwRL5s73Pn6IBOd+i8iMgFr1qxhw4YNPPnkk9x+++10d3czdepUotEozz//PAcPHhzz/jfccAOPP/44ANu3b2fbtm05q00jdBGRCVi8eDEnT55kxowZ1NTUcOedd3LrrbfS2NjIkiVLWLBgwZj3v++++7jnnntoaGhgyZIlLFu2LGe16fK5InLR0OVzdflcEZFLggJdRCQgFOgiclGZrGniC+3d9FOBLiIXjVgsRmdnZ+BD3d3p7OwkFotN6H46ykVELhp1dXW0tLRwKVytNRaLUVdXN6H7KNBF5KIRjUapr6+f7DLetzTlIiISEAp0EZGAUKCLiASEAl1EJCAU6CIiATFuoJvZo2bWZmZjfmiFmV1nZkkzG/mD80RE5LzKZoT+GLByrAZmFga+AfwyBzWJiMi7MG6gu/tG4Pg4zb4E/Ahoy0VRIiIyce95Dt3MZgCfBtZl0XatmTWZWdOlcKaXiMiFlIs3Rb8J3O/uyfEauvt6d29098bq6uocPLSIiLwjF6f+NwIbMp+RVwV8yswS7v7THOxbRESy9J4D3d1PX1jBzB4Dfq4wFxG58MYNdDN7AlgBVJlZC/B1IArg7uPOm4uIyIUxbqC7+x3Z7szdP/eeqhERkXdNZ4qKiASEAl1EJCAU6CIiAaFAFxEJCAW6iEhAKNBFRAJCgS4iEhAKdBGRgFCgi4gEhAJdRCQgFOgiIgGhQBcRCQgFuohIQCjQRUQCQoEuIhIQ4wa6mT1qZm1mtn2U7Xea2bbM10tmdnXuyxQRkfFkM0J/DFg5xvb9wI3u3gD8HbA+B3WJiMgEZfOJRRvNbM4Y218atrgJqHvvZYmIyETleg7988AvRttoZmvNrMnMmtrb23P80CIil7acBbqZfZR0oN8/Wht3X+/uje7eWF1dnauHFhERsphyyYaZNQCPAKvcvTMX+xQRkYl5zyN0M5sF/Bj4Y3ff/d5LEhGRd2PcEbqZPQGsAKrMrAX4OhAFcPd1wNeASuBhMwNIuHvj+SpYRERGls1RLneMs/1PgT/NWUUiIvKu6ExREZGAUKCLiASEAl1EJCAU6CIiAaFAFxEJCAW6iEhAKNBFRAJCgS4iEhAKdBGRgFCgi4gEhAJdRCQgFOgiIgGhQBcRCQgFuohIQCjQRUQCYtxAN7NHzazNzLaPst3M7CEz22tm28xsae7LFBGR8WQzQn8MWDnG9lXAvMzXWuDb770sERGZqHED3d03AsfHaLIa+BdP2wRMMbOaXBUoIiLZycUc+gygedhyS2adiIhcQLkIdBthnY/Y0GytmTWZWVN7e3sOHlpERN6Ri0BvAWYOW64DWkdq6O7r3b3R3Rurq6tz8NAiIvKOXAT608DdmaNdlgPd7n4kB/sVEZEJiIzXwMyeAFYAVWbWAnwdiAK4+zrgGeBTwF6gD7jnfBUrIiKjGzfQ3f2OcbY78Oc5q0hERN4VnSkqIhIQCnQRkYBQoIuIBIQCXUQkIBToIiIBoUAXEQkIBbqISEAo0EVEAkKBLiISEAp0EZGAUKCLiASEAl1EJCAU6CIiAaFAFxEJCAW6iEhAKNBFRAIiq0A3s5Vm9paZ7TWzB0bYXmZmPzOz181sh5npU4tERC6wcQPdzMLAt4BVwCLgDjNbdFazPwd2uvvVpD+u7h/MLC/HtYqIyBiyGaEvA/a6+z53HwI2AKvPauNAiZkZUAwcBxI5rVRERMaUTaDPAJqHLbdk1g33z8BCoBV4A/gLd0+dvSMzW2tmTWbW1N7e/i5LFhGRkWQT6DbCOj9r+ZPAVqAWWAL8s5mVnnMn9/Xu3ujujdXV1RMuVkRERpdNoLcAM4ct15EeiQ93D/BjT9sL7AcW5KZEERHJRjaBvhmYZ2b1mTc61wBPn9XmEHATgJlNA64A9uWyUBERGVtkvAbunjCzLwK/BMLAo+6+w8zuzWxfB/wd8JiZvUF6iuZ+d+84j3WLiMhZxg10AHd/BnjmrHXrht1uBT6R29JERGQidKaoiEhAKNBFRAJCgS4iEhAKdBGRgFCgi4gEhAJdRCQgFOgiIgGhQBcRCQgFuohIQCjQRUQCQoEuIhIQCnQRkYBQoIuIBIQCXUQkIBToIiIBkVWgm9lKM3vLzPaa2QOjtFlhZlvNbIeZvZjbMkVEZDzjfsCFmYWBbwG/R/rzRTeb2dPuvnNYmynAw8BKdz9kZlPPV8EiIjKybEboy4C97r7P3YeADcDqs9r8EekPiT4E4O5tuS1TRETGk02gzwCahy23ZNYNNx8oN7MXzGyLmd090o7MbK2ZNZlZU3t7+7urWERERpRNoNsI6/ys5QhwLXAz8Engb8xs/jl3cl/v7o3u3lhdXT3hYkVEZHTZfEh0CzBz2HId0DpCmw537wV6zWwjcDWwOydViojIuLIZoW8G5plZvZnlAWuAp89q8xTwETOLmFkhcD2wK7eliojIWMYdobt7wsy+CPwSCAOPuvsOM7s3s32du+8ys2eBbUAKeMTdt5/PwkVE5EzmfvZ0+IXR2NjoTU1Nk/LYIiIXKzPb4u6NI23TmaIiIgGhQBcRCQgFuohIQCjQRUQCQoEuIhIQCnQRkYBQoIuIBIQCXUQkIBToIiIBoUAXEQkIBbqISEAo0EVEAkKBLiISEAp0EZGAUKCLiAREVoFuZivN7C0z22tmD4zR7jozS5rZ7bkrUUREsjFuoJtZGPgWsApYBNxhZotGafcN0p9sJCIiF1g2I/RlwF533+fuQ8AGYPUI7b4E/Ahoy2F9IiKSpWwCfQbQPGy5JbPuNDObAXwaWDfWjsxsrZk1mVlTe3v7RGsVEZExZBPoNsK6sz+I9JvA/e6eHGtH7r7e3RvdvbG6ujrbGkVEJAuRLNq0ADOHLdcBrWe1aQQ2mBlAFfApM0u4+09zUqWIiIwrm0DfDMwzs3rgMLAG+KPhDdy9/p3bZvYY8HOFuYjIhTVuoLt7wsy+SProlTDwqLvvMLN7M9vHnDcXEZELI5sROu7+DPDMWetGDHJ3/9x7L0tERCZKZ4qKiASEAl1EJCAU6CIiAaFAFxEJCAW6iEhAKNBFRAJCgS4iEhAKdBGRgFCgi4gEhAJdRCQgFOgiIgGhQBcRCQgFuohIQCjQRUQCQoEuIhIQWQW6ma00s7fMbK+ZPTDC9jvNbFvm6yUzuzr3pYqIyFjGDXQzCwPfAlYBi4A7zGzRWc32Aze6ewPwd8D6XBcqIiJjy2aEvgzY6+773H0I2ACsHt7A3V9y9xOZxU2kP0haREQuoGwCfQbQPGy5JbNuNJ8HfvFeihIRkYnL5jNFbYR1PmJDs4+SDvQPj7J9LbAWYNasWVmWKCIi2chmhN4CzBy2XAe0nt3IzBqAR4DV7t450o7cfb27N7p7Y3V19bupV0RERpFNoG8G5plZvZnlAWuAp4c3MLNZwI+BP3b33bkvU0RExjPulIu7J8zsi8AvgTDwqLvvMLN7M9vXAV8DKoGHzQwg4e6N569sERE5m7mPOB1+3jU2NnpTU9OkPLaIyMXKzLaMNmDWmaIiIgGhQBcRCQgFuohIQCjQRUQCQoEuIhIQCnQRkYBQoIuIBIQCXUQkIBToIiIBoUAXEQkIBbqISEAo0EVEAkKBLiISEAp0EZGAUKCLiASEAl1EJCCyCnQzW2lmb5nZXjN7YITtZmYPZbZvM7OluS9VRETGMm6gm1kY+BawClgE3GFmi85qtgqYl/laC3w7x3WKiMg4shmhLwP2uvs+dx8CNgCrz2qzGvgXT9sETDGzmhzXKiIiYxj3Q6KBGUDzsOUW4Pos2swAjgxvZGZrSY/gAQbNbPuEqr04VQEdk13EBXKp9FX9DJaLrZ+zR9uQTaDbCOvO/mTpbNrg7uuB9QBm1jTaB50GyaXST7h0+qp+BkuQ+pnNlEsLMHPYch3Q+i7aiIjIeZRNoG8G5plZvZnlAWuAp89q8zRwd+Zol+VAt7sfOXtHIiJy/ow75eLuCTP7IvBLIAw86u47zOzezPZ1wDPAp4C9QB9wTxaPvf5dV31xuVT6CZdOX9XPYAlMP839nKluERG5COlMURGRgFCgi4gExKQE+niXEriYmNmjZtY2/Jh6M6sws1+Z2Z7M9/Jh2/4q0++3zOyTk1P1xJnZTDN73sx2mdkOM/uLzPpA9dXMYmb2ipm9nunng5n1gernO8wsbGavmdnPM8tB7ecBM3vDzLaaWVNmXfD66u4X9Iv0G6tvA5cBecDrwKILXUcO+3MDsBTYPmzd/wIeyNx+APhG5vaiTH/zgfrMzyE82X3Isp81wNLM7RJgd6Y/geor6XMqijO3o8BvgeVB6+ew/v434HvAzzPLQe3nAaDqrHWB6+tkjNCzuZTARcPdNwLHz1q9GvhO5vZ3gN8ftn6Duw+6+37SRwUtuyCFvkfufsTdX83cPgnsIn02cKD66mmnMovRzJcTsH4CmFkdcDPwyLDVgevnGALX18kI9NEuExAk0zxzHH7m+9TM+kD03czmANeQHr0Grq+ZaYitQBvwK3cPZD+BbwJ/CaSGrQtiPyH9pPxvZrYlcwkSCGBfszn1P9eyukxAQF30fTezYuBHwFfcvcdspC6lm46w7qLoq7sngSVmNgX4iZldOUbzi7KfZnYL0ObuW8xsRTZ3GWHd+76fw3zI3VvNbCrwKzN7c4y2F21fJ2OEfilcJuDYO1ebzHxvy6y/qPtuZlHSYf64u/84szqQfQVw9y7gBWAlwevnh4DbzOwA6WnPj5nZdwlePwFw99bM9zbgJ6SnUALX18kI9GwuJXCxexr4k8ztPwGeGrZ+jZnlm1k96evHvzIJ9U2YpYfi/xfY5e7/OGxToPpqZtWZkTlmVgB8HHiTgPXT3f/K3evcfQ7p/8F/d/e7CFg/AcysyMxK3rkNfALYTgD7OlnvOH+K9FESbwN/PdnvDL/HvjxB+jLBcdLP7J8HKoFfA3sy3yuGtf/rTL/fAlZNdv0T6OeHSb/s3AZszXx9Kmh9BRqA1zL93A58LbM+UP08q88r+N1RLoHrJ+kj6l7PfO14J3OC2Fed+i8iEhA6U1REJCAU6CIiAaFAFxEJCAW6iEhAKItEPf0AAAAVSURBVNBFRAJCgS4iEhAKdBGRgPj/1UJSWnLxkBQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.fit_one_cycle(1, max_lr=slice(lr*0.95**num_groups, lr), moms=(0.8, 0.9))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner.save('third_cycle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "seed_all(seed)\n",
    "learner.load('third_cycle');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner.unfreeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>error_rate</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.693604</td>\n",
       "      <td>1.725655</td>\n",
       "      <td>0.230694</td>\n",
       "      <td>0.769306</td>\n",
       "      <td>00:58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1.627851</td>\n",
       "      <td>1.695488</td>\n",
       "      <td>0.268817</td>\n",
       "      <td>0.731183</td>\n",
       "      <td>00:55</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXhc9X3v8fd3RqNdsqzFtmzZlg3ewRgQYAiLCQmxWWyS0sSsCQ11yNIs7b2FPFkIN+ltSJs+SZoAoSkhpCSUEsIWloQGcG/MJgdsvOB9k1dZsrVZ6+h7/5ixkOSRNLLHlnX4vJ5Hz8w553fO+f1G0uec+Z3N3B0RERn+QkNdARERSQ0FuohIQCjQRUQCQoEuIhIQCnQRkYBIG6oVh7NHeG5RKVNG5w5VFUREhp3ly5fvd/eSRNOGLNDTRoxi5A3fp/K7Vw5VFUREhh0z29bXtCHvctnf2DrUVRARCYQhD/SN+xqHugoiIoEw5IG+vfbQUFdBRCQQhqwP/bDtNQp0EUlOe3s7VVVVtLS0DHVVjrvMzEzKysqIRCJJzzPkgb5hX8NQV0FEhomqqiry8vIoLy/HzIa6OseNu1NTU0NVVRWTJk1Ker4hDfTzJhXy8rpq2jo6SU8b8t4fEajdAj9fAKEIhCMQTodwWvw1HUKH33eb3lU28t60UCRBuW7LOTw+Ubke4/tYdygNAhxofWlpaQl8mAOYGUVFRVRXVw9qviEN9IunlvD6llq21jQxdXRej2lvbq2ltb2TC04pIhQK9i9PTiKRLJjyYYi2Q7Qt/hp/3xl/39783rTOw+U6jhzX2XF869pjYzLARqLHxuR4bKDS+qhPr3KhY99xC3qYH3Y07RzSQJ9dNgKAdXsamDo6j/94bRtff2IVZ4wvYMWOgwB88vyJ3LXotKGspryf5I2Bhf+ammW5H7kx6L6R6Ey00Ti8Yei+keg+PsEGpvty+1pmRwu01r+3zL7q09EKHMdbalvoKDcm8eHJn4aD2wCLLQuLf1Pp9Zpo3OF5+pw20PwnvyEN9Bml+YRDxmPLqzh4qI1vPLkagBU7DpKeFmL6mDwefn07V5xeynmTi3p0zeyrb6EgO11dNQHUEe0kZHZM38z21LXwzSdXMbYgi9suOYXm9ig//uNGWtqj5GWm8fFzxjOzNJ/bf7OStbvrmV1WwLTReYzMSae8KJulG/Zz43kTONQWZXxhNiE7ij0mM0hLj/0MJ53RJDYSg9iY9N5AJdqY9LXMjpae805ohZYGwGMbzO6vx1Us1A/WNfCrJ57jc5+6LokNQAgMIMQVn7iFX/3sRxSMKEh+wwPQfgjWPd/zW1V/tRyqB1xklE7xhh3rmPr153qM/4uzyhgzIoNPXlBOJBTi2vuWsam6idPG5bNuTwOXzxzD7LIR/ONz7wLw+Ocu4O8eXcHM0nxmjs1nb30LRTkZFOel84mK8aSFgx/47s6La/eRkxHmglOKcXfe2VnHi2v2curoPObPGjNsNnxL11dz8wNvkBUJ88kLyll4xliqG1vJTAsxoSibzLQwI3Pe+6N2d6obWnn49e3UNbdz49yJfOyeP1HfktrujvGFWVw+cwyLzxlPS3snEwqzGZHd8+yDlvYo6eGQugiPo7Vr1zJjxowjJ/QO9+6vA07rTHr+rdu2c9UnPsWqV/8Qn9YJONGODsLh0MDr673egdq7bR8zXvh4j3F2V/1yd69IVH5IA71l13quuWdZV/fKZy6ZzFcX9Pxlbd3fxIIf/g/N7dGklpudHuZQW6zs2BGZ/PNfnsHInHTGjsiirrmd3Mw0CnNOvj0md6fTYUftIQpz08nPfC8sXt9cQ01TG/XN7Xz/D+u58NRiPn/pKUQ74ccvbeTtHQfYUdsMwKyx+TS1drC11+mgv/ns+Zw1YWTSe5nNbVEyIyGinc722kNMKMw+YuPo7v0u71BbBz98cQMzSvM5/5QiSnIzMIv9Le+ub6G+uZ31exuYM76AiUU5/NvSzfzDs2sByEkP09SW+Hd+2rh8zi0vYnJJDv+1vKrr76e7OeML+IePnsbe+hb+6sFKAD5RMZ6zJ45kRHaEz/xyOelpIb66YDo3zZ3I1ppDZKeHWbq+ml0HmyktyOK/1+6loryQt7cfZGtNE+/u6XlGVmYkxMfOKuNrV8zgoVe3cc9LG2mNdnLrhZOYNiaPVzfVcPv86T02QHJs+gz0E2Tx4sU8+eSTTJs2jUgkQm5uLqWlpbz99tusWbOGa665hh07dtDS0sKXvvQllixZAkB5eTmVlZU0NjayYMECLrzwQpYtW8a4ceN48onfkpWZmTD4165bx4yR0R7fgOzUy06+QM8aO9Wbd61nf2MrDS0dRDs7OaUkN2FAuDurdtYzvjCLv7h3GY2tHZw2dgR/ffFkHnp1KzfNLWfu5EIOtUXJTg+zv7GN51bt5ocvbqCmqa3HskIGHz2zjA9OH0V2ephxI7PYebCZS6aU0NIR5ZkVu3llfTXLtx3gXz5xBhecUnzcP4u6Q+3c8uAb/Hl7LJhK8jL43x+ZxoisCP9VuYMX1+47Yp7MSIiW9tgWfu7kQs6cMJLXNtfw1vaDjMiKcN25E7h0Wglv7TjId+PfZgC+fuUMbjp/Iunh0BGf9cZ9jdz3yiaqG1p5ZX3Po+uj8jL4x4+dztkTR7J6Vz3/8Lu1bNnfxKI5Y/niZVOo3HaAOWUFTCjK7prn87/6M79buTupz+Ajs0bzwuq9APz8U+dw0ZRiHly2lT11LTy4bCufm3cK6WkhHn9rJ5urm7rmy8tMY3R+JnctnEXVgUO8tf0g86aV8OGZYwjH95TrmtsJGeR120i2RzvpiDpZ6eGk6gfw5Ns7ue+VzXxoxih21B7i9S217K5773zozEiIjLQwdc3tPeb7wKlFfO2Kmcwcm9/17Wn93kbe2FJDU2uU08tGUN/czj0vb2La6Dz+8zNzKcjWRiCR7oF+19OrWbOrPqXLnzk2nzuvntXn9K1bt3LVVVexatUqXn75Za688kpWrVrVdWphbW0thYWFNDc3c8455/DKK69QVFTUI9BPPfVUKisrmTNnDh//+MdZuHAhN95444DtPczMTr5Anzxjtm9eu3LQ8w20V9jd3voWXli9h//ZsJ8po3I51Bbl9S21rN195B9BRlqIcMi69u4B0kLG1NF5fGjGKOZMKODSaaMwM+pb2vFO2F3fTFNrlLMnjuyxrPZoJ+v2NFA2MqvHP+aKHQdZUXWQ4twMJpfkMK4giyff3sW9L29i58FmSvIymD4mj7W769nf2NZVry9eNoWC7Ag/fWUzdy2cxfjCLL7zu7Ws2lnHN6+excIzxnat482ttRTnZjCpOKdr3I7aQ3z0nmU97pszszSfn950NuMLs2lpj9LcFuWSf3qpR1fFqaNy+701Q3Fuelc9DztjfAEHD7Vx9oSRPP7WTv7y7DKmjcnjqRW7CIeMt+IbrbzMNKaPySNkRkYkzIodB7loSjHfvGomo/Iz+1wnxO7/s2pnHRMKsxN+cziR3thSyy9f28a55SO5ce5EOjqdtbvrWVFVR0e0k7ueXtNVduroXJpao+w82NzvMicWZfOJc8Zz2fTRjMiKUJAdITOS/IYnyE62QL/rrrt46aWXuqZ/61vf4re//W1X2RdeeIG5c+f2CPQPf/jDbNiwAYC7776b9vZ2vv71rydc32ADfcgOih5tt8dgDkyNzs/k5vPLufn88q5xnZ3O0yt30RF1MiIhXttcw4a9jby+pRaAy2eO5ouXTWFEVoQfvLiB36/ew4/+uLFr/gtOKWLZppoe6xlXkMXnLj2FMfmZFGRH+OuHllMb/2ZQlJNOQXaEsyaM5Ldv7aSj88gN6LiCLO654SyuOL0UgLaOTv5r+Q6Wbazh29ec1vVZ3XDexK55Hrzl3IRtPqe88Ihx4wuzefWrH+TgoXa+9tt3yE4P88d393HR9146ouyPrz8TIP4NJvbn0RHtZEVVHb98dStRh1NLcrn+vAkU5aTzz79fR9SdS6eN4l9+v543tsY+x23xLow7F84iNyONWy+a3LWOlvboMQVUcW4G86aNOur5U+ncSYWcO+m9zzwSNmaXFTC7rACAT55fzotr93LnU6tZv7eRi6YUc/UZY8nNCHPDeRN55p3d1DS2MrYgi8tnjuZvH13BH9/dx/eeX8f3nl8HxPrvn/7Chazf20jpiEzGF2YfUY9nVu6iODeDuZOLBlX/1o4odc3trN/TyNQxuYzK67kx7ex0Wjs6B/VN5kTpL3hPlJyc93acXn75ZV588UVeffVVsrOzmTdv3hFXtLZHOwlH0tld10xBVoRwOExzc/8b+MEY8itFT7RQyFg0Z1zX8FWzY3u3tU1t7G9s7XE+/Pc/fgbr906moaWdp1fs5ukVu1i+7QClIzJpjzpnTywgMxLmz9sP8LXfruqxnlNKcjhvchFvbT/I2t31bKpuYv6sMfzd5VOpOtDM1pomnl+1hymjc7lr4Wld3QMA6WkhbjhvYo8AP1aRcIiSvAzuvzm2Yd+yv4lPPvBG1710PnBqEVfPHtv1eXSXFg5x9sSRR3wTAfj7+dO73j962/k0t0VJCxsvrtlLcV4GuRlH/om9n/Y2QyHj8lljuGhKCY2tHZTkZfSYftPcnr/jn950NpuqG9mwt5HVu+ppaY/yi1e3Muf//KGrTMXEkfz1xZNZseMgT769CzOoOvBeKEwsyqa1vZNzJhXy7UWzenxLbO2I8uCftvKHNXspyI6woqqO6ob3vrl9+5rTuGnuRNqjnfxu5W5++N8b2LK/iUumlnD3X8xmzIj+vz0FXV5eHg0NsWMp0c6eBzXr6uoYOXIk2dnZvPvuu7z22mvUNLaycV8DnQ7ba5rYf6CRjmgn1Q2tVDe00tIeJZW9JEPW5VJRUeGVlZVDsu5Ua2mPsnpXHev2NLKpupHPXDy5R7fBvvoWou6UjsgawlomNpguLBkaP//TFr7/+/XMKM1jR20ze+oHdx+TaaPzqCgfyZc+NIVbf1HJyqq6rmmFOenMLM1ny/4minLTWVlVx6I5YxmTn8lPl27usRwz+F+XT+PMCQWcP7nohP7d7Gto4Ym3dnL2iEMUjptMXmYa6eEQuZlphPqpRzT+jTjc68yj5rYOGlo7KM7JSHhWUrTTj5jnsOuvv54VK1diaekUF4/i2WefIWRG1f56br3xE1Tv3c2M6dPZuWcvt37p7znn/AtZcP5sfvW7l6C9hdtu+jj/740/s/NgM7+47185dKiJb3zzTtLTQqSHQ+RkhON1Dg2fPvQgBbrIiXKorYP1exvZtK+R8uIczhxfwItr95KVHmbKqDxyMsI0t0cZlZfJ65tr+NbTa2hsbWfXwZaucDulJIfb50/nkmklhMyIxI9B1B1q5/IfvMLe+tgee+mITF7820vIyUjrOp30sMKcdD5wajHr9zSwbm8D5UXZ3HR+OX/1geQuy9+yv4nGlg7+tGk/JbkZfOyscV3ztUc7ebRyBxlpYTbua+Q/39zOgUOxA83/trCU0RPe677LyUhjYh/HUdo6omzY2wgGY0Zk0h6/jqWuuYOm1g463UkPh8iMhMmIhKhvbic7PY1wyKhpbCU7I43MSBjjveuKGls6kj7jDmBEVoTcjDRCIcPiw93buXV/Ey0dnQn30iPhELU7t7A1WsCM0nzmxLvxwuGQAl3k/ezP2w/w3WffZc6EAu6YP73Pc+X3NbSwamcdq3fWc+PciT1OuXynqo5XN++noaWDf40fVxqTn0ltUxvtnZ24x04bPm3sCK44fQwTi3M4o6yAwpx03J22aCcvr6vmqbd38bt3ep79NGVULiV5GVxxein/9j+b2ZbgLqxXzi5lyekZnDp1Gmkho6mto6urKTs9jZHZEaKdzojsCB1RZ1vNITo6E5/rnRYOkZN+5BlJ3RmG93HB0uj8zK4TKbbsbyIcMsoKssjLirDrYOxkidH5GT0CvC+d7rR1dNLcFqW6MdYNEzIjNyONTRvW8ekn3/usImFj4/+98ugD3cweAK4C9rl7wmvwzWwe8AMgAux390v6XSgKdJHhrCPaSXVjK2PyMzEz2jo6eeLtnbyxpZbKrbVd10HMGpvPbz4bu/ive4inp4X47CWnMCo/g3eq6njkzR09lj+xKHYGU35WhL/6wCTmjC8gHLIjuiDqmtvZXtOUMHbTwiHGFWQSCceupzCMg4fayM1M6wraA4faiEYds9jeflrIqG/pIC8zDQP2NrSSkx7GgUOtUTIiIYpy0o9rd1P3btDVq9ewL1zMa5tr+P2avXxoxii+ftWsYwr0i4FG4KFEgW5mBcAyYL67bzezUe5+5InTvSjQRYKpuS3Kd59by+tbao+4GOuq2aWUF+Xw6Qsnde39uzs7DzYzIivCm1traetwPjJrdMLQTNSn3NnpNLV1UN/SQUtblKa2DopzMyjJy+jqThquUn7aorsvNbPyfopcDzzu7tvj5QcMcxEJrqz0MHctOg135+HXt7O7rpmKiYVcOj3xqaZmRtnI2KmYH5w+etDrC4WMvMxIjwvH3q9ScdriVCBiZi8DecAP3f2hRAXNbAmwBGDChAkpWLWInKzMjBvnpu7UWxlYKr6PpAFnA1cCHwG+YWZTExV09/vdvcLdK0pKSlKwahEROSwVgV4FPO/uTe6+H1gKnJGC5YqIDGu5ubkA7Nq1i2uvvTZhmXnz5pGq44mpCPQngYvMLM3MsoHzgLUpWK6ISCCMHTuWxx577LivZ8A+dDP7NTAPKDazKuBOYqcn4u73uftaM3seWEnsBr8/c/dVfS1PRGS4uv3225k4cSKf+9zngNjNuMyMpUuXcuDAAdrb2/nOd77DokWLeszX/aZezc3N3HLLLaxZs4YZM2ac2Hu5uPt1SZT5J+CfUlIjEZFkPHcH7Hkntcscczos+G6fkxcvXsyXv/zlrkB/9NFHef755/nKV75Cfn4++/fvZ+7cuSxcuLDPc9XvvfdesrOzWblyJStXruSss85KWfXfdzfnEhE5WmeeeSb79u1j165dVFdXM3LkSEpLS/nKV77C0qVLCYVC7Ny5k7179zJmzJiEy1i6dClf/OIXAZg9ezazZ89OWf0U6CIyPPWzJ308XXvttTz22GPs2bOHxYsX8/DDD1NdXc3y5cuJRCKUl5cfcdvc3o7XlabD+zIqEZETbPHixTzyyCM89thjXHvttdTV1TFq1CgikQgvvfQS27Zt63f+iy++mIcffhiAVatWsXLl4B/00xftoYuIDMKsWbNoaGhg3LhxlJaWcsMNN3D11VdTUVHBnDlzmD59er/zf/azn+WWW25h9uzZzJkzh3PPTfywmqOhuy2KyLAx1A+JPtEGey8XdbmIiASEAl1EJCAU6CIyrAxVN/GJdjTtVKCLyLCRmZlJTU1N4EPd3ampqSEzc3AP5dZZLiIybJSVlVFVVUV1dfVQV+W4y8zMpKysbFDzKNBFZNiIRCJMmjRpqKtx0lKXi4hIQCjQRUQCQoEuIhIQCnQRkYAYMNDN7AEz22dm/T60wszOMbOomSV+zpKIiBxXyeyhPwjM76+AmYWBu4EXUlAnERE5CgMGursvBWoHKPY3wG+AfamolIiIDN4x96Gb2Tjgo8B9SZRdYmaVZlb5frgwQETkRErFQdEfALe7e3Sggu5+v7tXuHtFSUlJClYtIiKHpeJK0QrgkfgjlYqBK8ysw92fSMGyRUQkSccc6O7edR2umT0IPKMwFxE58QYMdDP7NTAPKDazKuBOIALg7gP2m4uIyIkxYKC7+3XJLszdP3VMtRERkaOmK0VFRAJCgS4iEhAKdBGRgFCgi4gEhAJdRCQgFOgiIgGhQBcRCQgFuohIQCjQRUQCQoEuIhIQCnQRkYBQoIuIBIQCXUQkIBToIiIBoUAXEQkIBbqISEAMGOhm9oCZ7TOzVX1Mv8HMVsZ/lpnZGamvpoiIDCSZPfQHgfn9TN8CXOLus4FvA/enoF4iIjJIyTyCbqmZlfczfVm3wdeAsmOvloiIDFaq+9A/DTzX10QzW2JmlWZWWV1dneJVi4i8v6Us0M3sUmKBfntfZdz9fnevcPeKkpKSVK1aRERIosslGWY2G/gZsMDda1KxTBERGZxj3kM3swnA48BN7r7+2KskIiJHY8A9dDP7NTAPKDazKuBOIALg7vcB3wSKgHvMDKDD3SuOV4VFRCSxZM5yuW6A6bcCt6asRiIiclR0paiISEAo0EVEAkKBLiISEAp0EZGAUKCLiASEAl1EJCAU6CIiAaFAFxEJCAW6iEhAKNBFRAJCgS4iEhAKdBGRgFCgi4gEhAJdRCQgFOgiIgExYKCb2QNmts/MVvUx3czsR2a20cxWmtlZqa+miIgMJJk99AeB+f1MXwBMif8sAe499mqJiMhgDRjo7r4UqO2nyCLgIY95DSgws9JUVVBERJKTij70ccCObsNV8XFHMLMlZlZpZpXV1dUpWLWIiByWikC3BOM8UUF3v9/dK9y9oqSkJAWrFhGRw1IR6FXA+G7DZcCuFCxXREQGIRWB/hRwc/xsl7lAnbvvTsFyRURkENIGKmBmvwbmAcVmVgXcCUQA3P0+4FngCmAjcAi45XhVVkRE+jZgoLv7dQNMd+DzKauRiIgcFV0pKiISEAp0EZGAUKCLiASEAl1EJCAU6CIiAaFAFxEJCAW6iEhAKNBFRAJCgS4iEhAKdBGRgFCgi4gEhAJdRCQgFOgiIgGhQBcRCQgFuohIQCQV6GY238zWmdlGM7sjwfQRZva0ma0ws9VmpodciIicYAMGupmFgZ8AC4CZwHVmNrNXsc8Da9z9DGJPN/q+maWnuK4iItKPZPbQzwU2uvtmd28DHgEW9SrjQJ6ZGZAL1AIdKa2piIj0K5lAHwfs6DZcFR/X3Y+BGcAu4B3gS+7e2XtBZrbEzCrNrLK6uvooqywiIokkE+iWYJz3Gv4I8DYwFpgD/NjM8o+Yyf1+d69w94qSkpJBV1ZERPqWTKBXAeO7DZcR2xPv7hbgcY/ZCGwBpqemiiIikoxkAv1NYIqZTYof6FwMPNWrzHbgMgAzGw1MAzansqIiItK/tIEKuHuHmX0BeAEIAw+4+2ozuy0+/T7g28CDZvYOsS6a2919/3Gst4iI9DJgoAO4+7PAs73G3dft/S7g8tRWTUREBkNXioqIBIQCXUQkIBToIiIBoUAXEQkIBbqISEAo0EVEAkKBLiISEAp0EZGAUKCLiASEAl1EJCAU6CIiAaFAFxEJCAW6iEhAKNBFRAJCgS4iEhAKdBGRgEgq0M1svpmtM7ONZnZHH2XmmdnbZrbazF5JbTVFRGQgAz6xyMzCwE+ADxN7YPSbZvaUu6/pVqYAuAeY7+7bzWzU8aqwiIgklswe+rnARnff7O5twCPAol5lrgced/ftAO6+L7XVFBGRgSQT6OOAHd2Gq+LjupsKjDSzl81suZndnGhBZrbEzCrNrLK6uvroaiwiIgklE+iWYJz3Gk4DzgauBD4CfMPMph4xk/v97l7h7hUlJSWDrqyIiPRtwD50Ynvk47sNlwG7EpTZ7+5NQJOZLQXOANanpJYiIjKgZPbQ3wSmmNkkM0sHFgNP9SrzJHCRmaWZWTZwHrA2tVUVEZH+DLiH7u4dZvYF4AUgDDzg7qvN7Lb49Pvcfa2ZPQ+sBDqBn7n7quNZcRER6cnce3eHnxgVFRVeWVk5JOsWERmuzGy5u1ckmqYrRUVEAkKBLiISEAp0EZGAUKCLiASEAl1EJCAU6CIiAaFAFxEJCAW6iEhAKNBFRAJCgS4iEhAKdBGRgFCgi4gEhAJdRCQgFOgiIgGhQBcRCYikAt3M5pvZOjPbaGZ39FPuHDOLmtm1qauiiIgkY8BAN7Mw8BNgATATuM7MZvZR7m5iTzYSEZETLJk99HOBje6+2d3bgEeARQnK/Q3wG2BfCusnIiJJSibQxwE7ug1Xxcd1MbNxwEeB+/pbkJktMbNKM6usrq4ebF1FRKQfyQS6JRjX+0GkPwBud/dofwty9/vdvcLdK0pKSpKto4iIJCEtiTJVwPhuw2XArl5lKoBHzAygGLjCzDrc/YmU1FJERAaUTKC/CUwxs0nATmAxcH33Au4+6fB7M3sQeEZhLiJyYg0Y6O7eYWZfIHb2Shh4wN1Xm9lt8en99puLiMiJkcweOu7+LPBsr3EJg9zdP3Xs1RIRkcHSlaIiIgGhQBcRCQgFuohIQCjQRUQCQoEuIhIQCnQRkYBQoIuIBIQCXUQkIBToIiIBoUAXEQkIBbqISEAo0EVEAkKBLiISEAp0EZGAUKCLiAREUoFuZvPNbJ2ZbTSzOxJMv8HMVsZ/lpnZGamvqoiI9GfAQDezMPATYAEwE7jOzGb2KrYFuMTdZwPfBu5PdUVFRKR/yeyhnwtsdPfN7t4GPAIs6l7A3Ze5+4H44GvEHiQtIiInUDKBPg7Y0W24Kj6uL58Gnks0wcyWmFmlmVVWV1cnX0sRERlQMoFuCcZ5woJmlxIL9NsTTXf3+929wt0rSkpKkq+liIgMKJmHRFcB47sNlwG7ehcys9nAz4AF7l6TmuqJiEiyktlDfxOYYmaTzCwdWAw81b2AmU0AHgducvf1qa+miIgMZMA9dHfvMLMvAC8AYeABd19tZrfFp98HfBMoAu4xM4AOd684ftUWEZHezD1hd/hxV1FR4ZWVlUOybhGR4crMlve1w6wrRUVEAkKBLiISEAp0EZGAUKCLiASEAl1EJCAU6CIiAaFAFxEJCAW6iEhAKNBFRAJCgS4iEhAKdBGRgFCgi4gEhAJdRCQgFOgiIgGhQBcRCQgFuohIQCQV6GY238zWmdlGM7sjwXQzsx/Fp680s7NSX1UREenPgIFuZmHgJ8ACYCZwnZnN7FVsATAl/rMEuDfF9RQRkQEks4d+LrDR3Te7exvwCLCoV5lFwEMe8xpQYGalKa6riIj0Y8CHRAPjgB3dhquA85IoMw7Y3b2QmS0htgcP0GpmqwZV2+GpGNg/1JU4Qd4vbVU7g2W4tXNiXxOSCXRLMK73k6WTKYO73w/cD2BmlX096DRI3i/thPdPW9XOYAlSO5PpcqkCxncbLgN2HUUZEcGrapcAAAPmSURBVBE5jpIJ9DeBKWY2yczSgcXAU73KPAXcHD/bZS5Q5+67ey9IRESOnwG7XNy9w8y+ALwAhIEH3H21md0Wn34f8CxwBbAROATcksS67z/qWg8v75d2wvunrWpnsASmneZ+RFe3iIgMQ7pSVEQkIBToIiIBMSSBPtCtBIYTMxtvZi+Z2VozW21mX4qPLzSzP5jZhvjryG7zfDXe9nVm9pGhq/3gmVnYzN4ys2fiw4Frp5kVmNljZvZu/Pd6fkDb+ZX43+wqM/u1mWUGpZ1m9oCZ7et+rcvRtM3Mzjazd+LTfmRmiU7RPnm4+wn9IXZgdRMwGUgHVgAzT3Q9UtieUuCs+Ps8YD2xWyR8D7gjPv4O4O74+5nxNmcAk+KfRXio2zGI9v4t8Cvgmfhw4NoJ/AK4Nf4+HSgIWjuJXfi3BciKDz8KfCoo7QQuBs4CVnUbN+i2AW8A5xO71uY5YMFQt62/n6HYQ0/mVgLDhrvvdvc/x983AGuJ/bMsIhYMxF+vib9fBDzi7q3uvoXYmUHnnthaHx0zKwOuBH7WbXSg2mlm+cTC4N8B3L3N3Q8SsHbGpQFZZpYGZBO7diQQ7XT3pUBtr9GDalv89iX57v6qx9L9oW7znJSGItD7uk3AsGdm5cCZwOvAaI+fix9/HRUvNpzb/wPg74HObuOC1s7JQDXw83jX0s/MLIeAtdPddwL/DGwndouOOnf/PQFrZy+Dbdu4+Pve409aQxHoSd0mYLgxs1zgN8CX3b2+v6IJxp307Tezq4B97r482VkSjDvp20lsr/Us4F53PxNoIvb1vC/Dsp3x/uNFxLoYxgI5ZnZjf7MkGHfStzNJfbVt2LV5KAI9cLcJMLMIsTB/2N0fj4/ee/iOk/HXffHxw7X9HwAWmtlWYt1kHzSz/yB47awCqtz99fjwY8QCPmjt/BCwxd2r3b0deBy4gOC1s7vBtq0q/r73+JPWUAR6MrcSGDbiR73/HVjr7v/SbdJTwCfj7z8JPNlt/GIzyzCzScTuIf/Giarv0XL3r7p7mbuXE/ud/dHdbyR47dwD7DCzafFRlwFrCFg7iXW1zDWz7Pjf8GXEjv8ErZ3dDapt8W6ZBjObG/+Mbu42z8lpiI5AX0HsbJBNwNeG+sjwMbblQmJfw1YCb8d/rgCKgP8GNsRfC7vN87V429dxkh8176PN83jvLJfAtROYA1TGf6dPACMD2s67gHeBVcAviZ3lEYh2Ar8mdmygndie9qePpm1ARfzz2QT8mPjV9Sfrjy79FxEJCF0pKiISEAp0EZGAUKCLiASEAl1EJCAU6CIiAaFAFxEJCAW6iEhA/H/yPEjjTJqS/AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.fit_one_cycle(2, max_lr=slice(lr*0.95**num_groups, lr), moms=(0.8, 0.9))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Category tensor(1),\n",
       " tensor(1),\n",
       " tensor([0.1148, 0.3387, 0.1260, 0.1180, 0.1331, 0.1693]))"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.predict('This is the best movie of 2020')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Category tensor(1),\n",
       " tensor(1),\n",
       " tensor([0.1293, 0.3434, 0.1269, 0.1074, 0.1554, 0.1376]))"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.predict('This is the worst movie of 2020')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner.export(file = 'transformer.pkl');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "path = '/kaggle/working'\n",
    "export_learner = load_learner(path, file = 'transformer.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def get_preds_as_nparray(ds_type) -> np.ndarray:\n",
    "    \"\"\"\n",
    "    the get_preds method does not yield the elements in order by default\n",
    "    we borrow the code from the RNNLearner to resort the elements into their correct order\n",
    "    \"\"\"\n",
    "    preds = learner.get_preds(ds_type)[0].detach().cpu().numpy()\n",
    "    sampler = [i for i in databunch.dl(ds_type).sampler]\n",
    "    reverse_sampler = np.argsort(sampler)\n",
    "    return preds[reverse_sampler, :]\n",
    "\n",
    "test_preds = get_preds_as_nparray(DatasetType.Test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.19521621"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_preds[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub = pd.read_csv('../input/mh-20-fakenews/Participants_Data_WH20/sample submission.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0    1    2    3    4    5\n",
       "0  0.0  0.0  0.0  0.0  0.0  0.0\n",
       "1  0.0  0.0  0.0  0.0  0.0  0.0\n",
       "2  0.0  0.0  0.0  0.0  0.0  0.0\n",
       "3  0.0  0.0  0.0  0.0  0.0  0.0\n",
       "4  0.0  0.0  0.0  0.0  0.0  0.0"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for i in range(0,len(sub)):\n",
    "    for j in range(0,6):\n",
    "        sub.iloc[i,j] = test_preds[i][j]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1267.000000</td>\n",
       "      <td>1267.000000</td>\n",
       "      <td>1267.000000</td>\n",
       "      <td>1267.000000</td>\n",
       "      <td>1267.000000</td>\n",
       "      <td>1267.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.154710</td>\n",
       "      <td>0.167917</td>\n",
       "      <td>0.224634</td>\n",
       "      <td>0.219698</td>\n",
       "      <td>0.064371</td>\n",
       "      <td>0.168670</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.072234</td>\n",
       "      <td>0.069655</td>\n",
       "      <td>0.047047</td>\n",
       "      <td>0.093708</td>\n",
       "      <td>0.071262</td>\n",
       "      <td>0.093366</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.018618</td>\n",
       "      <td>0.032466</td>\n",
       "      <td>0.051385</td>\n",
       "      <td>0.011541</td>\n",
       "      <td>0.002232</td>\n",
       "      <td>0.010246</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.098151</td>\n",
       "      <td>0.110249</td>\n",
       "      <td>0.194325</td>\n",
       "      <td>0.146127</td>\n",
       "      <td>0.018069</td>\n",
       "      <td>0.099320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.143973</td>\n",
       "      <td>0.161916</td>\n",
       "      <td>0.225135</td>\n",
       "      <td>0.209276</td>\n",
       "      <td>0.039370</td>\n",
       "      <td>0.152884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.197863</td>\n",
       "      <td>0.219281</td>\n",
       "      <td>0.255865</td>\n",
       "      <td>0.293837</td>\n",
       "      <td>0.084269</td>\n",
       "      <td>0.229666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.397343</td>\n",
       "      <td>0.370256</td>\n",
       "      <td>0.370951</td>\n",
       "      <td>0.465044</td>\n",
       "      <td>0.640687</td>\n",
       "      <td>0.659271</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 0            1            2            3            4  \\\n",
       "count  1267.000000  1267.000000  1267.000000  1267.000000  1267.000000   \n",
       "mean      0.154710     0.167917     0.224634     0.219698     0.064371   \n",
       "std       0.072234     0.069655     0.047047     0.093708     0.071262   \n",
       "min       0.018618     0.032466     0.051385     0.011541     0.002232   \n",
       "25%       0.098151     0.110249     0.194325     0.146127     0.018069   \n",
       "50%       0.143973     0.161916     0.225135     0.209276     0.039370   \n",
       "75%       0.197863     0.219281     0.255865     0.293837     0.084269   \n",
       "max       0.397343     0.370256     0.370951     0.465044     0.640687   \n",
       "\n",
       "                 5  \n",
       "count  1267.000000  \n",
       "mean      0.168670  \n",
       "std       0.093366  \n",
       "min       0.010246  \n",
       "25%       0.099320  \n",
       "50%       0.152884  \n",
       "75%       0.229666  \n",
       "max       0.659271  "
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub.to_csv('MH20_FastAI_v1a.csv', index = False)# lb: 1.6918133822685584"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
